[OPIK-473] Add Daily report

apps/opik-backend/config.yml

@@ -82,9 +82,13 @@ rateLimit:
 usageReport:
   enabled: ${OPIK_USAGE_REPORT_ENABLED:-true}
   url: ${OPIK_USAGE_REPORT_URL:-https://stats.comet.com/notify/event/}
+  serverStats:
+    enabled: ${OPIK_USAGE_REPORT_SERVER_STATS_ENABLED:-true}
 
 metadata:
   version: ${OPIK_VERSION:-latest}
 
 cors:
   enabled: ${CORS:-false}
+
+stateDatabaseName: ${STATE_DB_DATABASE_NAME:-opik}

apps/opik-backend/pom.xml

@@ -141,6 +141,11 @@
             <artifactId>guicey-jdbi3</artifactId>
             <version>${dropwizard-guicey.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.github.dropwizard-jobs</groupId>
+            <artifactId>dropwizard-jobs-guice</artifactId>
+            <version>6.0.0</version>
+        </dependency>
 
         <dependency>
             <groupId>org.jdbi</groupId>

apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java

@@ -3,13 +3,15 @@
 import com.comet.opik.infrastructure.ConfigurationModule;
 import com.comet.opik.infrastructure.OpikConfiguration;
 import com.comet.opik.infrastructure.auth.AuthModule;
+import com.comet.opik.infrastructure.bi.BiModule;
 import com.comet.opik.infrastructure.bi.OpikGuiceyLifecycleEventListener;
 import com.comet.opik.infrastructure.bundle.LiquibaseBundle;
 import com.comet.opik.infrastructure.db.DatabaseAnalyticsModule;
 import com.comet.opik.infrastructure.db.IdGeneratorModule;
 import com.comet.opik.infrastructure.db.NameGeneratorModule;
 import com.comet.opik.infrastructure.events.EventModule;
 import com.comet.opik.infrastructure.http.HttpModule;
+import com.comet.opik.infrastructure.job.JobGuiceyInstaller;
 import com.comet.opik.infrastructure.ratelimit.RateLimitModule;
 import com.comet.opik.infrastructure.redis.RedisModule;
 import com.comet.opik.utils.JsonBigDecimalDeserializer;
@@ -37,6 +39,8 @@
 
 public class OpikApplication extends Application<OpikConfiguration> {
 
+    private GuiceBundle guiceBundle;
+
     public static void main(String[] args) throws Exception {
         new OpikApplication().run(args);
     }
@@ -61,19 +65,24 @@ public void initialize(Bootstrap<OpikConfiguration> bootstrap) {
                 .migrationsFileName(DB_APP_ANALYTICS_MIGRATIONS_FILE_NAME)
                 .dataSourceFactoryFunction(OpikConfiguration::getDatabaseAnalyticsMigrations)
                 .build());
-        bootstrap.addBundle(GuiceBundle.builder()
+
+        guiceBundle = GuiceBundle.builder()
                 .bundles(JdbiBundle.<OpikConfiguration>forDatabase((conf, env) -> conf.getDatabase())
                         .withPlugins(new SqlObjectPlugin(), new Jackson2Plugin()))
                 .modules(new DatabaseAnalyticsModule(), new IdGeneratorModule(), new AuthModule(), new RedisModule(),
                         new RateLimitModule(), new NameGeneratorModule(), new HttpModule(), new EventModule(),
-                        new ConfigurationModule())
+                        new ConfigurationModule(), new BiModule())
+                .installers(JobGuiceyInstaller.class)
                 .listen(new OpikGuiceyLifecycleEventListener())
                 .enableAutoConfig()
-                .build());
+                .build();
+
+        bootstrap.addBundle(guiceBundle);
     }
 
     @Override
     public void run(OpikConfiguration configuration, Environment environment) {
+
         // Resources
         var jersey = environment.jersey();
 

apps/opik-backend/src/main/java/com/comet/opik/domain/DatasetDAO.java

@@ -133,12 +133,12 @@ int[] recordExperiments(@Bind("workspace_id") String workspaceId,
             "FROM datasets " +
             "WHERE created_at BETWEEN DATE_SUB(CURDATE(), INTERVAL 1 DAY) AND CURDATE() " +
             "GROUP BY workspace_id,created_by")
-    List<BiInformationResponse.BiInformation> getExperimentBIInformation();
+    List<BiInformationResponse.BiInformation> getDatasetsBIInformation();
 
     @SqlUpdate("CREATE TEMPORARY TABLE experiment_dataset_ids_<table_name> (id CHAR(36) PRIMARY KEY)")
     void createTempTable(@Define("table_name") String tableName);
 
-    @SqlBatch("INSERT INTO experiment_dataset_ids_<table_name> (id) VALUES (:id)")
+    @SqlBatch("INSERT INTO experiment_dataset_ids_<table_name>(id) VALUES (:id)")
     int[] insertTempTable(@Define("table_name") String tableName, @Bind("id") List<UUID> id);
 
     @SqlUpdate("DROP TEMPORARY TABLE IF EXISTS experiment_dataset_ids_<table_name>")

apps/opik-backend/src/main/java/com/comet/opik/domain/DatasetService.java

@@ -74,6 +74,8 @@ public interface DatasetService {
     BiInformationResponse getDatasetBIInformation();
 
     Set<UUID> exists(Set<UUID> datasetIds, String workspaceId);
+
+    long getDailyCreatedCount();
 }
 
 @Singleton
@@ -380,7 +382,7 @@ public BiInformationResponse getDatasetBIInformation() {
         log.info("Getting dataset BI events daily data");
         return template.inTransaction(READ_ONLY, handle -> {
             var dao = handle.attach(DatasetDAO.class);
-            var biInformation = dao.getExperimentBIInformation();
+            var biInformation = dao.getDatasetsBIInformation();
             return BiInformationResponse.builder()
                     .biInformation(biInformation)
                     .build();
@@ -488,4 +490,16 @@ public Mono<Void> recordExperiments(Set<DatasetLastExperimentCreated> datasetsLa
                 .then();
     }
 
+    @Override
+    @WithSpan
+    public long getDailyCreatedCount() {
+        return template.inTransaction(READ_ONLY, handle -> {
+            var dao = handle.attach(DatasetDAO.class);
+            return dao.getDatasetsBIInformation()
+                    .stream()
+                    .mapToLong(BiInformationResponse.BiInformation::count)
+                    .sum();
+        });
+    }
+
 }

apps/opik-backend/src/main/java/com/comet/opik/domain/ExperimentDAO.java

@@ -730,4 +730,10 @@ private Publisher<ExperimentDatasetId> mapDatasetId(Result result) {
         return result.map((row, rowMetadata) -> new ExperimentDatasetId(row.get("dataset_id", UUID.class)));
     }
 
+    public Mono<Long> getDailyCreatedCount() {
+        return Mono.from(connectionFactory.create())
+                .flatMapMany(connection -> connection.createStatement(EXPERIMENT_DAILY_BI_INFORMATION).execute())
+                .flatMap(result -> result.map((row, rowMetadata) -> row.get("experiment_count", Long.class)))
+                .reduce(0L, Long::sum);
+    }
 }

apps/opik-backend/src/main/java/com/comet/opik/domain/ExperimentService.java

@@ -319,6 +319,7 @@ public Flux<DatasetLastExperimentCreated> getMostRecentCreatedExperimentFromData
         return experimentDAO.getMostRecentCreatedExperimentFromDatasets(datasetIds);
     }
 
+    @WithSpan
     public Mono<BiInformationResponse> getExperimentBIInformation() {
         log.info("Getting experiment BI events daily data");
         return experimentDAO.getExperimentBIInformation()
@@ -330,4 +331,9 @@ public Mono<BiInformationResponse> getExperimentBIInformation() {
                 .switchIfEmpty(Mono.just(BiInformationResponse.empty()));
 
     }
+
+    @WithSpan
+    public Mono<Long> getDailyCreatedCount() {
+        return experimentDAO.getDailyCreatedCount();
+    }
 }

apps/opik-backend/src/main/java/com/comet/opik/domain/TraceDAO.java

@@ -83,6 +83,8 @@ interface TraceDAO {
     Flux<BiInformation> getTraceBIInformation(Connection connection);
 
     Mono<ProjectStats> getStats(TraceSearchCriteria criteria);
+
+    Mono<Long> getDailyTraces();
 }
 
 @Slf4j
@@ -1125,6 +1127,15 @@ public Mono<ProjectStats> getStats(@NonNull TraceSearchCriteria criteria) {
         });
     }
 
+    @Override
+    public Mono<Long> getDailyTraces() {
+        return asyncTemplate
+                .nonTransaction(
+                        connection -> Mono.from(connection.createStatement(TRACE_COUNT_BY_WORKSPACE_ID).execute()))
+                .flatMapMany(result -> result.map((row, rowMetadata) -> row.get("trace_count", Long.class)))
+                .reduce(0L, Long::sum);
+    }
+
     @Override
     @WithSpan
     public Mono<Map<UUID, Instant>> getLastUpdatedTraceAt(

apps/opik-backend/src/main/java/com/comet/opik/domain/TraceService.java

@@ -66,6 +66,8 @@ public interface TraceService {
     Mono<BiInformationResponse> getTraceBIInformation();
 
     Mono<ProjectStats> getStats(TraceSearchCriteria searchCriteria);
+
+    Mono<Long> getDailyCreatedCount();
 }
 
 @Slf4j
@@ -207,11 +209,10 @@ private Mono<UUID> create(Trace trace, Project project, UUID id) {
 
     private Mono<Project> handleProjectCreationError(Throwable exception, String projectName, String workspaceId) {
         return switch (exception) {
-            case EntityAlreadyExistsException __ ->
-                Mono.fromCallable(
-                        () -> projectService.findByNames(workspaceId, List.of(projectName)).stream().findFirst()
-                                .orElseThrow())
-                        .subscribeOn(Schedulers.boundedElastic());
+            case EntityAlreadyExistsException __ -> Mono.fromCallable(
+                    () -> projectService.findByNames(workspaceId, List.of(projectName)).stream().findFirst()
+                            .orElseThrow())
+                    .subscribeOn(Schedulers.boundedElastic());
             default -> Mono.error(exception);
         };
     }
@@ -352,6 +353,7 @@ public Mono<BiInformationResponse> getTraceBIInformation() {
                 .switchIfEmpty(Mono.just(BiInformationResponse.empty()));
     }
 
+    @Override
     @WithSpan
     public Mono<ProjectStats> getStats(@NonNull TraceSearchCriteria criteria) {
 
@@ -361,8 +363,14 @@ public Mono<ProjectStats> getStats(@NonNull TraceSearchCriteria criteria) {
         }
 
         return getProjectByName(criteria.projectName())
-                .flatMap(project -> template.nonTransaction(
-                        connection -> dao.getStats(criteria.toBuilder().projectId(project.id()).build())))
+                .flatMap(project -> dao.getStats(criteria.toBuilder().projectId(project.id()).build()))
                 .switchIfEmpty(Mono.just(ProjectStats.empty()));
     }
+
+    @Override
+    @WithSpan
+    public Mono<Long> getDailyCreatedCount() {
+        return dao.getDailyTraces();
+    }
+
 }

apps/opik-backend/src/main/java/com/comet/opik/infrastructure/OpikConfiguration.java

@@ -1,14 +1,14 @@
 package com.comet.opik.infrastructure;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import io.dropwizard.core.Configuration;
 import io.dropwizard.db.DataSourceFactory;
+import io.dropwizard.jobs.JobConfiguration;
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import lombok.Getter;
 
 @Getter
-public class OpikConfiguration extends Configuration {
+public class OpikConfiguration extends JobConfiguration {
 
     @Valid
     @NotNull @JsonProperty
@@ -53,4 +53,9 @@ public class OpikConfiguration extends Configuration {
     @Valid
     @NotNull @JsonProperty
     private BatchOperationsConfig batchOperations = new BatchOperationsConfig();
+
+    @Valid
+    @NotNull @JsonProperty
+    private String stateDatabaseName;
+
 }

apps/opik-backend/src/main/java/com/comet/opik/infrastructure/UsageReportConfig.java

@@ -2,11 +2,15 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotNull;
 import lombok.Data;
 
 @Data
 public class UsageReportConfig {
 
+    public record ServerStatsConfig(boolean enabled) {
+    }
+
     @Valid
     @JsonProperty
     private boolean enabled;
@@ -15,4 +19,8 @@ public class UsageReportConfig {
     @JsonProperty
     private String url;
 
+    @Valid
+    @NotNull @JsonProperty
+    private UsageReportConfig.ServerStatsConfig serverStats;
+
 }

apps/opik-backend/src/main/java/com/comet/opik/infrastructure/bi/StartupEvent.java → apps/opik-backend/src/main/java/com/comet/opik/infrastructure/bi/BiEvent.java

@@ -10,5 +10,5 @@
 @Builder(toBuilder = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
-record StartupEvent(String anonymousId, String eventType, Map<String, String> eventProperties) {
+record BiEvent(String anonymousId, String eventType, Map<String, String> eventProperties) {
 }

apps/opik-backend/src/main/java/com/comet/opik/infrastructure/bi/BiModule.java

@@ -0,0 +1,17 @@
+package com.comet.opik.infrastructure.bi;
+
+import com.comet.opik.infrastructure.OpikConfiguration;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import ru.vyarus.dropwizard.guice.module.support.DropwizardAwareModule;
+
+import static com.comet.opik.infrastructure.UsageReportConfig.ServerStatsConfig;
+
+public class BiModule extends DropwizardAwareModule<OpikConfiguration> {
+
+    @Provides
+    @Singleton
+    public ServerStatsConfig provideServerStatsConfig() {
+        return configuration().getUsageReport().getServerStats();
+    }
+}