feat(reports): sidecar container report generation (#779)

* feat(reports): implement/config sidecar container report generator

* Add remote report status to health

* extract some env var names that are referenced in multiple places

* apply report generation locking only in subprocess mode

* default to subprocess generation if env var unset

* apply limits to report generator

* error handling cleanup

* test(health): include reports status in expected output

* fixup! default to subprocess generation if env var unset

* apply spotless formatting

* fix(reports): throw expected exception type when recording not found in target

* fix(reports): attempt to open local file stream before remote recording stream

* suppress spurious warning

* change cryostat-reports namespace after repo transfer

* extract SslConfiguration variables

extract subprocess report generation variable

extract platform configuration variables

extract max WS connections variable

extract CORS origin variable

extract DISABLE_SSL variable

extract webserver configuration variables

use Variables

extract JMX connection config variables

finish extracting variables

fix missed import

fix extracted variables in tests

* remove unnecessary comment

* use xpath host var

* fixup! change cryostat-reports namespace after repo transfer

* remove unused class

* fix(reports): apply timeout when generating reports to avoid hanging requests

* fix(reports): reports handlers should be unordered

request ordering is handled deeper in the stack by the SubprocessReportGenerator synchronizing lock, or is handled by the configured sidecar report generator setup and its potential load balancer

* fix unit tests

* use latest cryostat-reports and configure for small footprint

* enable JDP and JMX on cryostat-reports sidecar

allows Cryostat too inspect the report generator for profiling and analysis

run.sh

@@ -98,9 +98,11 @@ podman run \
     --mount type=bind,source="$(dirname $0)/templates",destination=/opt/cryostat.d/templates.d,relabel=shared \
     --mount type=bind,source="$(dirname $0)/truststore",destination=/truststore,relabel=shared \
     --mount type=tmpfs,target=/opt/cryostat.d/probes.d \
+    -e CRYOSTAT_REPORT_GENERATOR=$CRYOSTAT_REPORT_GENERATOR \
     -e CRYOSTAT_PLATFORM=$CRYOSTAT_PLATFORM \
     -e CRYOSTAT_DISABLE_SSL=$CRYOSTAT_DISABLE_SSL \
     -e CRYOSTAT_DISABLE_JMX_AUTH=$CRYOSTAT_DISABLE_JMX_AUTH \
+    -e CRYOSTAT_ALLOW_UNTRUSTED_SSL=$CRYOSTAT_ALLOW_UNTRUSTED_SSL \
     -e CRYOSTAT_RJMX_USER=$CRYOSTAT_RJMX_USER \
     -e CRYOSTAT_RJMX_PASS=$CRYOSTAT_RJMX_PASS \
     -e CRYOSTAT_RJMX_PORT=$CRYOSTAT_RJMX_PORT \

smoketest.sh

@@ -12,6 +12,8 @@ function runCryostat() {
         GRAFANA_DASHBOARD_URL="http://${host}:${grafanaPort}" \
         CRYOSTAT_RJMX_USER=smoketest \
         CRYOSTAT_RJMX_PASS=smoketest \
+        CRYOSTAT_ALLOW_UNTRUSTED_SSL=true \
+        CRYOSTAT_REPORT_GENERATOR="http://${host}:10001" \
         exec "$DIR/run.sh"
 }
 
@@ -76,6 +78,19 @@ function runGrafana() {
         --rm -d "${stream}:${tag}"
 }
 
+function runReportGenerator() {
+    local RJMX_PORT=10000
+    podman run \
+        --name reports \
+        --pod cryostat \
+        --cpus 1 \
+        --memory 512M \
+        --restart on-failure \
+        --env JAVA_OPTIONS="-XX:ActiveProcessorCount=1 -XX:+UseSerialGC -Dorg.openjdk.jmc.flightrecorder.parser.singlethreaded=true -Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote.port=${RJMX_PORT} -Dcom.sun.management.jmxremote.rmi.port=${RJMX_PORT} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" \
+        --env QUARKUS_HTTP_PORT=10001 \
+        --rm -d quay.io/cryostat/cryostat-reports:latest
+}
+
 function createPod() {
     local jmxPort="$(xpath -q -e 'project/properties/cryostat.rjmxPort/text()' pom.xml)"
     local webPort="$(xpath -q -e 'project/properties/cryostat.webPort/text()' pom.xml)"
@@ -97,7 +112,9 @@ function createPod() {
         --publish 9999:9999 \
         --publish 8082:8082 \
         --publish 9990:9990 \
-        --publish 9991:9991
+        --publish 9991:9991 \
+        --publish 10000:10000 \
+        --publish 10001:10001
     # 8081: vertx-fib-demo
     # 9093: vertx-fib-demo-1 RJMX
     # 9094: vertx-fib-demo-2 RJMX
@@ -106,7 +123,9 @@ function createPod() {
     # 9999: quarkus-test HTTP
     # 8082: Wildfly HTTP
     # 9990: Wildfly Admin Console
-    # 9990: Wildfly RJMX
+    # 9991: Wildfly RJMX
+    # 10000: cryostat-reports RJMX
+    # 10001: cryostat-reports HTTP
 }
 
 function destroyPod() {
@@ -119,4 +138,5 @@ createPod
 runDemoApps
 runJfrDatasource
 runGrafana
+runReportGenerator
 runCryostat

src/main/java/io/cryostat/MainModule.java

@@ -46,6 +46,7 @@
 import javax.management.remote.JMXServiceURL;
 
 import io.cryostat.configuration.ConfigurationModule;
+import io.cryostat.configuration.Variables;
 import io.cryostat.core.log.Logger;
 import io.cryostat.core.sys.Environment;
 import io.cryostat.core.tui.ClientWriter;
@@ -144,7 +145,7 @@ JMXServiceURL.class, new GsonJmxServiceUrlAdapter(logger))
     @Singleton
     @Named(RECORDINGS_PATH)
     static Path provideSavedRecordingsPath(Logger logger, Environment env) {
-        String archivePath = env.getEnv("CRYOSTAT_ARCHIVE_PATH", "/flightrecordings");
+        String archivePath = env.getEnv(Variables.ARCHIVE_PATH, "/flightrecordings");
         logger.info("Local save path for flight recordings set as {}", archivePath);
         return Paths.get(archivePath);
     }

src/main/java/io/cryostat/configuration/ConfigurationModule.java

@@ -67,7 +67,7 @@ public abstract class ConfigurationModule {
     @Singleton
     @Named(CONFIGURATION_PATH)
     static Path provideConfigurationPath(Logger logger, Environment env) {
-        String path = env.getEnv("CRYOSTAT_CONFIG_PATH", "/opt/cryostat.d/conf.d");
+        String path = env.getEnv(Variables.CONFIG_PATH, "/opt/cryostat.d/conf.d");
         logger.info(String.format("Local config path set as %s", path));
         return Paths.get(path);
     }

src/main/java/io/cryostat/configuration/Variables.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright The Cryostat Authors
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or data
+ * (collectively the "Software"), free of charge and under any and all copyright
+ * rights in the Software, and any and all patent rights owned or freely
+ * licensable by each licensor hereunder covering either (i) the unmodified
+ * Software as contributed to or provided by such licensor, or (ii) the Larger
+ * Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software (each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ * The above copyright notice and either this complete permission notice or at
+ * a minimum a reference to the UPL must be included in all copies or
+ * substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package io.cryostat.configuration;
+
+public final class Variables {
+    private Variables() {}
+
+    // jfr-datasource, cryostat-grafana-dashboard
+    public static final String GRAFANA_DATASOURCE_ENV = "GRAFANA_DATASOURCE_URL";
+    public static final String GRAFANA_DASHBOARD_ENV = "GRAFANA_DASHBOARD_URL";
+
+    // report generation
+    public static final String REPORT_GENERATOR_ENV = "CRYOSTAT_REPORT_GENERATOR";
+    public static final String SUBPROCESS_MAX_HEAP_ENV = "CRYOSTAT_REPORT_GENERATION_MAX_HEAP";
+
+    // SSL configuration
+    public static final String DISABLE_SSL = "CRYOSTAT_DISABLE_SSL";
+    public static final String KEYSTORE_PATH_ENV = "KEYSTORE_PATH";
+    public static final String KEYSTORE_PASS_ENV = "KEYSTORE_PASS";
+    public static final String KEY_PATH_ENV = "KEY_PATH";
+    public static final String CERT_PATH_ENV = "CERT_PATH";
+
+    // platform configuration
+    public static final String PLATFORM_STRATEGY_ENV_VAR = "CRYOSTAT_PLATFORM";
+    public static final String AUTH_MANAGER_ENV_VAR = "CRYOSTAT_AUTH_MANAGER";
+
+    // webserver configuration
+    public static final String WEBSERVER_HOST = "CRYOSTAT_WEB_HOST";
+    public static final String WEBSERVER_PORT = "CRYOSTAT_WEB_PORT";
+    public static final String WEBSERVER_PORT_EXT = "CRYOSTAT_EXT_WEB_PORT";
+    public static final String WEBSERVER_SSL_PROXIED = "CRYOSTAT_SSL_PROXIED";
+    public static final String WEBSERVER_ALLOW_UNTRUSTED_SSL = "CRYOSTAT_ALLOW_UNTRUSTED_SSL";
+    public static final String MAX_CONNECTIONS_ENV_VAR = "CRYOSTAT_MAX_WS_CONNECTIONS";
+    public static final String ENABLE_CORS_ENV = "CRYOSTAT_CORS_ORIGIN";
+
+    // JMX connections configuration
+    public static final String TARGET_CACHE_SIZE = "CRYOSTAT_TARGET_CACHE_SIZE";
+    public static final String TARGET_CACHE_TTL = "CRYOSTAT_TARGET_CACHE_TTL";
+
+    // paths configuration
+    public static final String ARCHIVE_PATH = "CRYOSTAT_ARCHIVE_PATH";
+    public static final String CONFIG_PATH = "CRYOSTAT_CONFIG_PATH";
+}

src/main/java/io/cryostat/messaging/MessagingModule.java

@@ -43,6 +43,7 @@
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import io.cryostat.configuration.Variables;
 import io.cryostat.core.log.Logger;
 import io.cryostat.core.sys.Clock;
 import io.cryostat.core.sys.Environment;
@@ -64,8 +65,6 @@ public abstract class MessagingModule {
     static final String WS_MAX_CONNECTIONS = "WS_MAX_CONNECTIONS";
     static final String LIMBO_PRUNER = "LIMBO_PRUNER";
     static final String KEEPALIVE_PINGER = "KEEPALIVE_PINGER";
-
-    static final String MAX_CONNECTIONS_ENV_VAR = "CRYOSTAT_MAX_WS_CONNECTIONS";
     static final int MIN_CONNECTIONS = 1;
     static final int MAX_CONNECTIONS = 64;
     static final int DEFAULT_MAX_CONNECTIONS = 2;
@@ -103,7 +102,7 @@ static int provideWebSocketMaxConnections(Environment env, Logger logger) {
             int maxConn =
                     Integer.parseInt(
                             env.getEnv(
-                                    MAX_CONNECTIONS_ENV_VAR,
+                                    Variables.MAX_CONNECTIONS_ENV_VAR,
                                     String.valueOf(DEFAULT_MAX_CONNECTIONS)));
             if (maxConn > MAX_CONNECTIONS) {
                 logger.info("Requested maximum WebSocket connections {} is too large.", maxConn);

src/main/java/io/cryostat/net/NetworkConfiguration.java

@@ -40,6 +40,7 @@
 import java.net.SocketException;
 import java.net.UnknownHostException;
 
+import io.cryostat.configuration.Variables;
 import io.cryostat.core.sys.Environment;
 
 public class NetworkConfiguration {
@@ -53,7 +54,7 @@ public class NetworkConfiguration {
     }
 
     public String getWebServerHost() throws SocketException, UnknownHostException {
-        return env.getEnv("CRYOSTAT_WEB_HOST", resolver.getHostAddress());
+        return env.getEnv(Variables.WEBSERVER_HOST, resolver.getHostAddress());
     }
 
     public int getDefaultWebServerPort() {
@@ -62,19 +63,20 @@ public int getDefaultWebServerPort() {
 
     public int getInternalWebServerPort() {
         return Integer.parseInt(
-                env.getEnv("CRYOSTAT_WEB_PORT", String.valueOf(getDefaultWebServerPort())));
+                env.getEnv(Variables.WEBSERVER_PORT, String.valueOf(getDefaultWebServerPort())));
     }
 
     public int getExternalWebServerPort() {
         return Integer.parseInt(
-                env.getEnv("CRYOSTAT_EXT_WEB_PORT", String.valueOf(getInternalWebServerPort())));
+                env.getEnv(
+                        Variables.WEBSERVER_PORT_EXT, String.valueOf(getInternalWebServerPort())));
     }
 
     public boolean isSslProxied() {
-        return env.hasEnv("CRYOSTAT_SSL_PROXIED");
+        return env.hasEnv(Variables.WEBSERVER_SSL_PROXIED);
     }
 
     public boolean isUntrustedSslAllowed() {
-        return env.hasEnv("CRYOSTAT_ALLOW_UNTRUSTED_SSL");
+        return env.hasEnv(Variables.WEBSERVER_ALLOW_UNTRUSTED_SSL);
     }
 }

src/main/java/io/cryostat/net/NetworkModule.java

@@ -47,6 +47,7 @@
 import javax.inject.Singleton;
 
 import io.cryostat.configuration.ConfigurationModule;
+import io.cryostat.configuration.Variables;
 import io.cryostat.core.log.Logger;
 import io.cryostat.core.net.JFRConnectionToolkit;
 import io.cryostat.core.sys.Environment;
@@ -77,9 +78,6 @@
         })
 public abstract class NetworkModule {
 
-    static final String TARGET_CACHE_SIZE = "CRYOSTAT_TARGET_CACHE_SIZE";
-    static final String TARGET_CACHE_TTL = "CRYOSTAT_TARGET_CACHE_TTL";
-
     @Provides
     @Singleton
     static HttpServer provideHttpServer(
@@ -101,24 +99,24 @@ static NetworkResolver provideNetworkResolver() {
     }
 
     @Provides
-    @Named(TARGET_CACHE_SIZE)
+    @Named(Variables.TARGET_CACHE_SIZE)
     static int provideMaxTargetConnections(Environment env) {
-        return Integer.parseInt(env.getEnv(TARGET_CACHE_SIZE, "-1"));
+        return Integer.parseInt(env.getEnv(Variables.TARGET_CACHE_SIZE, "-1"));
     }
 
     @Provides
-    @Named(TARGET_CACHE_TTL)
+    @Named(Variables.TARGET_CACHE_TTL)
     static Duration provideMaxTargetTTL(Environment env) {
-        return Duration.ofSeconds(Integer.parseInt(env.getEnv(TARGET_CACHE_TTL, "10")));
+        return Duration.ofSeconds(Integer.parseInt(env.getEnv(Variables.TARGET_CACHE_TTL, "10")));
     }
 
     @Provides
     @Singleton
     static TargetConnectionManager provideTargetConnectionManager(
             Lazy<JFRConnectionToolkit> connectionToolkit,
             PlatformClient platformClient,
-            @Named(TARGET_CACHE_TTL) Duration maxTargetTtl,
-            @Named(TARGET_CACHE_SIZE) int maxTargetConnections,
+            @Named(Variables.TARGET_CACHE_TTL) Duration maxTargetTtl,
+            @Named(Variables.TARGET_CACHE_SIZE) int maxTargetConnections,
             Logger logger) {
         return new TargetConnectionManager(
                 connectionToolkit,

src/main/java/io/cryostat/net/SslConfiguration.java

@@ -39,6 +39,7 @@
 
 import java.nio.file.Path;
 
+import io.cryostat.configuration.Variables;
 import io.cryostat.core.log.Logger;
 import io.cryostat.core.sys.Environment;
 import io.cryostat.core.sys.FileSystem;
@@ -56,18 +57,13 @@ public class SslConfiguration {
 
     private final SslConfigurationStrategy strategy;
 
-    private static final String KEYSTORE_PATH_ENV = "KEYSTORE_PATH";
-    private static final String KEYSTORE_PASS_ENV = "KEYSTORE_PASS";
-    private static final String KEY_PATH_ENV = "KEY_PATH";
-    private static final String CERT_PATH_ENV = "CERT_PATH";
-
     SslConfiguration(Environment env, FileSystem fs, Logger logger)
             throws SslConfigurationException {
         this.env = env;
         this.fs = fs;
         this.logger = logger;
 
-        if (env.hasEnv("CRYOSTAT_DISABLE_SSL")) {
+        if (env.hasEnv(Variables.DISABLE_SSL)) {
             strategy = new NoSslStrategy();
             logger.info("Selected NoSSL strategy");
             return;
@@ -76,7 +72,7 @@ public class SslConfiguration {
         {
             Path path = obtainKeyStorePathIfSpecified();
             if (path != null) {
-                strategy = new KeyStoreStrategy(path, env.getEnv(KEYSTORE_PASS_ENV, ""));
+                strategy = new KeyStoreStrategy(path, env.getEnv(Variables.KEYSTORE_PASS_ENV, ""));
                 logger.info("Selected SSL KeyStore strategy with keystore {}", path.toString());
                 return;
             }
@@ -99,7 +95,7 @@ public class SslConfiguration {
         {
             Path path = discoverKeyStorePathInDefaultLocations();
             if (path != null) {
-                strategy = new KeyStoreStrategy(path, env.getEnv(KEYSTORE_PASS_ENV, ""));
+                strategy = new KeyStoreStrategy(path, env.getEnv(Variables.KEYSTORE_PASS_ENV, ""));
                 logger.info("Selected SSL KeyStore strategy in default location");
                 return;
             }
@@ -128,11 +124,11 @@ public class SslConfiguration {
     }
 
     Path obtainKeyStorePathIfSpecified() throws SslConfigurationException {
-        if (!env.hasEnv(KEYSTORE_PATH_ENV)) {
+        if (!env.hasEnv(Variables.KEYSTORE_PATH_ENV)) {
             return null;
         }
 
-        Path path = fs.pathOf(env.getEnv(KEYSTORE_PATH_ENV)).normalize();
+        Path path = fs.pathOf(env.getEnv(Variables.KEYSTORE_PATH_ENV)).normalize();
         if (!fs.exists(path)) {
             throw new SslConfigurationException(
                     String.format(
@@ -144,16 +140,16 @@ Path obtainKeyStorePathIfSpecified() throws SslConfigurationException {
     }
 
     Pair<Path, Path> obtainKeyCertPathPairIfSpecified() throws SslConfigurationException {
-        if (!env.hasEnv(KEY_PATH_ENV) && !env.hasEnv(CERT_PATH_ENV)) {
+        if (!env.hasEnv(Variables.KEY_PATH_ENV) && !env.hasEnv(Variables.CERT_PATH_ENV)) {
             return null;
         }
 
-        if (env.hasEnv(KEY_PATH_ENV) ^ env.hasEnv(CERT_PATH_ENV)) {
+        if (env.hasEnv(Variables.KEY_PATH_ENV) ^ env.hasEnv(Variables.CERT_PATH_ENV)) {
             throw new SslConfigurationException("both KEY_PATH and CERT_PATH must be specified");
         }
 
-        Path key = fs.pathOf(env.getEnv(KEY_PATH_ENV)).normalize();
-        Path cert = fs.pathOf(env.getEnv(CERT_PATH_ENV)).normalize();
+        Path key = fs.pathOf(env.getEnv(Variables.KEY_PATH_ENV)).normalize();
+        Path cert = fs.pathOf(env.getEnv(Variables.CERT_PATH_ENV)).normalize();
         if (!fs.exists(key)) {
             throw new SslConfigurationException(
                     String.format(