#518 added finishReasons parameter to IngestionHistory services

gbif · Sep 5, 2023 · f7d531e · f7d531e
1 parent 898332b
commit f7d531e
Show file tree

Hide file tree

Showing 8 changed files with 82 additions and 36 deletions.
diff --git a/pom.xml b/pom.xml
@@ -84,7 +84,7 @@
     <spring-cloud-sleuth.version>2.2.8.RELEASE</spring-cloud-sleuth.version>
 
     <!-- GBIF -->
-    <gbif-api.version>1.12.1</gbif-api.version>
+    <gbif-api.version>1.12.2-SNAPSHOT</gbif-api.version>
     <gbif-common.version>0.59</gbif-common.version>
     <gbif-common-mybatis.version>1.3</gbif-common-mybatis.version>
     <gbif-common-ws.version>1.25</gbif-common-ws.version>

diff --git a/...stence/src/main/java/org/gbif/registry/persistence/mapper/DatasetProcessStatusMapper.java b/...stence/src/main/java/org/gbif/registry/persistence/mapper/DatasetProcessStatusMapper.java
@@ -13,37 +13,42 @@
  */
 package org.gbif.registry.persistence.mapper;
 
+import org.apache.ibatis.annotations.Param;
 import org.gbif.api.model.common.paging.Pageable;
 import org.gbif.api.model.crawler.DatasetProcessStatus;
+import org.gbif.api.model.crawler.FinishReason;
+import org.springframework.stereotype.Repository;
 
+import javax.annotation.Nullable;
 import java.util.List;
 import java.util.UUID;
 
-import javax.annotation.Nullable;
-
-import org.apache.ibatis.annotations.Param;
-import org.springframework.stereotype.Repository;
-
 /** Mapper that perform operations on {@link DatasetProcessStatus} instances. */
 @Repository
 public interface DatasetProcessStatusMapper {
 
-  int count();
+  int count(@Nullable @Param("finishReasons") List<FinishReason> finishReasons);
 
   int countAborted();
 
-  int countByDataset(@Param("datasetKey") UUID datasetKey);
+  int countByDataset(
+      @Param("datasetKey") UUID datasetKey,
+      @Nullable @Param("finishReasons") List<FinishReason> finishReasons);
 
   void create(DatasetProcessStatus datasetProcessStatus);
 
   void update(DatasetProcessStatus datasetProcessStatus);
 
   DatasetProcessStatus get(@Param("datasetKey") UUID datasetKey, @Param("attempt") int attempt);
 
-  List<DatasetProcessStatus> list(@Nullable @Param("page") Pageable page);
+  List<DatasetProcessStatus> list(
+      @Nullable @Param("finishReasons") List<FinishReason> finishReasons,
+      @Nullable @Param("page") Pageable page);
 
   List<DatasetProcessStatus> listByDataset(
-      @Param("datasetKey") UUID datasetKey, @Nullable @Param("page") Pageable page);
+      @Param("datasetKey") UUID datasetKey,
+      @Nullable @Param("finishReasons") List<FinishReason> finishReasons,
+      @Nullable @Param("page") Pageable page);
 
   List<DatasetProcessStatus> listAborted(@Nullable @Param("page") Pageable page);
 }
diff --git a/...ce/src/main/resources/org/gbif/registry/persistence/mapper/DatasetProcessStatusMapper.xml b/...ce/src/main/resources/org/gbif/registry/persistence/mapper/DatasetProcessStatusMapper.xml
@@ -104,6 +104,14 @@
   <select id="list" resultMap="DATASET_PROCESS_STATUS_MAP" parameterType="org.gbif.api.model.common.paging.Pageable">
     SELECT <include refid="DATASET_PROCESS_STATUS_FIELDS"/>
     FROM crawl_history
+    <where>
+      <if test="finishReasons != null and finishReasons.size > 0">
+        finish_reason IN
+        <foreach item="item" collection="finishReasons" open="(" separator="," close=")">
+          #{item,jdbcType=OTHER}
+        </foreach>
+      </if>
+    </where>
     ORDER BY started_crawling DESC
     <if test="page != null" >
       LIMIT #{page.limit} OFFSET #{page.offset}
@@ -113,12 +121,26 @@
   <select id="count" resultType="Integer">
     SELECT COUNT(*)
     FROM crawl_history
+    <where>
+      <if test="finishReasons != null and finishReasons.size > 0">
+        finish_reason IN
+        <foreach item="item" collection="finishReasons" open="(" separator="," close=")">
+          #{item,jdbcType=OTHER}
+        </foreach>
+      </if>
+    </where>
   </select>
 
   <select id="listByDataset" resultMap="DATASET_PROCESS_STATUS_MAP" parameterType="org.gbif.api.model.common.paging.Pageable">
     SELECT <include refid="DATASET_PROCESS_STATUS_FIELDS"/>
     FROM crawl_history
     WHERE dataset_key = #{datasetKey,jdbcType=OTHER}
+    <if test="finishReasons != null and finishReasons.size > 0">
+      AND finish_reason IN
+      <foreach item="item" collection="finishReasons" open="(" separator="," close=")">
+        #{item,jdbcType=OTHER}
+      </foreach>
+    </if>
     ORDER BY attempt DESC
     <if test="page != null" >
       LIMIT #{page.limit} OFFSET #{page.offset}
@@ -129,6 +151,12 @@
     SELECT COUNT(*)
     FROM crawl_history
     WHERE dataset_key = #{datasetKey,jdbcType=OTHER}
+    <if test="finishReasons != null and finishReasons.size > 0">
+      AND finish_reason IN
+      <foreach item="item" collection="finishReasons" open="(" separator="," close=")">
+        #{item,jdbcType=OTHER}
+      </foreach>
+    </if>
   </select>
 
 

diff --git a/...nes/src/main/java/org/gbif/registry/pipelines/DefaultRegistryIngestionHistoryService.java b/...nes/src/main/java/org/gbif/registry/pipelines/DefaultRegistryIngestionHistoryService.java
@@ -16,21 +16,22 @@
 import org.gbif.api.model.common.paging.Pageable;
 import org.gbif.api.model.common.paging.PagingResponse;
 import org.gbif.api.model.crawler.DatasetProcessStatus;
+import org.gbif.api.model.crawler.FinishReason;
 import org.gbif.api.model.pipelines.IngestionProcess;
 import org.gbif.api.model.pipelines.PipelineProcess;
 import org.gbif.api.model.registry.Dataset;
 import org.gbif.registry.persistence.mapper.DatasetMapper;
 import org.gbif.registry.persistence.mapper.DatasetProcessStatusMapper;
 import org.gbif.registry.persistence.mapper.pipelines.PipelineProcessMapper;
+import org.springframework.stereotype.Service;
 
+import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-import org.springframework.stereotype.Service;
-
 @Service
 public class DefaultRegistryIngestionHistoryService implements RegistryIngestionHistoryService {
 
@@ -48,23 +49,25 @@ public DefaultRegistryIngestionHistoryService(
   }
 
   @Override
-  public PagingResponse<IngestionProcess> ingestionHistory(Pageable pageable) {
+  public PagingResponse<IngestionProcess> ingestionHistory(
+      @Nullable List<FinishReason> finishReasons, Pageable pageable) {
     List<IngestionProcess> ingestions =
-        datasetProcessStatusMapper.list(pageable).stream()
+        datasetProcessStatusMapper.list(finishReasons, pageable).stream()
             .map(this::toIngestionProcess)
             .collect(Collectors.toList());
 
-    long count = datasetProcessStatusMapper.count();
+    long count = datasetProcessStatusMapper.count(finishReasons);
 
     return new PagingResponse<>(pageable, count, ingestions);
   }
 
   @Override
-  public PagingResponse<IngestionProcess> ingestionHistory(UUID datasetKey, Pageable pageable) {
+  public PagingResponse<IngestionProcess> ingestionHistory(
+      UUID datasetKey, @Nullable List<FinishReason> finishReasons, Pageable pageable) {
     // get datasets process statuses
     List<DatasetProcessStatus> datasetProcessStatuses =
-        datasetProcessStatusMapper.listByDataset(datasetKey, pageable);
-    long count = datasetProcessStatusMapper.countByDataset(datasetKey);
+        datasetProcessStatusMapper.listByDataset(datasetKey, finishReasons, pageable);
+    long count = datasetProcessStatusMapper.countByDataset(datasetKey, finishReasons);
 
     if (datasetProcessStatuses.isEmpty()) {
       return new PagingResponse<>(pageable, count);

diff --git a/...-pipelines/src/main/java/org/gbif/registry/pipelines/RegistryIngestionHistoryService.java b/...-pipelines/src/main/java/org/gbif/registry/pipelines/RegistryIngestionHistoryService.java
@@ -15,15 +15,20 @@
 
 import org.gbif.api.model.common.paging.Pageable;
 import org.gbif.api.model.common.paging.PagingResponse;
+import org.gbif.api.model.crawler.FinishReason;
 import org.gbif.api.model.pipelines.IngestionProcess;
 
+import javax.annotation.Nullable;
+import java.util.List;
 import java.util.UUID;
 
 public interface RegistryIngestionHistoryService {
 
-  PagingResponse<IngestionProcess> ingestionHistory(Pageable pageable);
+  PagingResponse<IngestionProcess> ingestionHistory(
+      @Nullable List<FinishReason> finishReasons, Pageable pageable);
 
-  PagingResponse<IngestionProcess> ingestionHistory(UUID datasetKey, Pageable pageable);
+  PagingResponse<IngestionProcess> ingestionHistory(
+      UUID datasetKey, @Nullable List<FinishReason> finishReasons, Pageable pageable);
 
   IngestionProcess getIngestionProcess(UUID datasetKey, int attempt);
 }
diff --git a/...es/src/main/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingService.java b/...es/src/main/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingService.java
@@ -231,7 +231,7 @@ PagingResponse<SearchResult> search(
   /**
    * Sends email to data administrator about absent identifiers issue with a dataset
    *
-   * <p>Deprecated: use {@link #notifyAbsentIdentifiers(UUID, int, String)} instead.
+   * <p>Deprecated: use {@link #notifyAbsentIdentifiers(UUID, int, long, String)} instead.
    *
    * @param datasetKey dataset key
    * @param attempt attempt to run

diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java
@@ -1456,7 +1456,9 @@ public DatasetProcessStatus getDatasetProcessStatus(
   @Override
   public PagingResponse<DatasetProcessStatus> listDatasetProcessStatus(Pageable page) {
     return new PagingResponse<>(
-        page, (long) datasetProcessStatusMapper.count(), datasetProcessStatusMapper.list(page));
+        page,
+        (long) datasetProcessStatusMapper.count(null),
+        datasetProcessStatusMapper.list(null, page));
   }
 
   @Hidden
@@ -1485,8 +1487,8 @@ public PagingResponse<DatasetProcessStatus> listDatasetProcessStatus(
       @PathVariable UUID key, Pageable page) {
     return new PagingResponse<>(
         page,
-        (long) datasetProcessStatusMapper.countByDataset(key),
-        datasetProcessStatusMapper.listByDataset(key, page));
+        (long) datasetProcessStatusMapper.countByDataset(key, null),
+        datasetProcessStatusMapper.listByDataset(key, null, page));
   }
 
   @Override

diff --git a/...y-ws/src/main/java/org/gbif/registry/ws/resources/pipelines/IngestionHistoryResource.java b/...y-ws/src/main/java/org/gbif/registry/ws/resources/pipelines/IngestionHistoryResource.java
@@ -13,22 +13,20 @@
  */
 package org.gbif.registry.ws.resources.pipelines;
 
+import io.swagger.v3.oas.annotations.Hidden;
 import org.gbif.api.model.common.paging.Pageable;
 import org.gbif.api.model.common.paging.PagingResponse;
+import org.gbif.api.model.crawler.FinishReason;
 import org.gbif.api.model.pipelines.IngestionProcess;
 import org.gbif.api.service.pipelines.IngestionHistoryService;
 import org.gbif.registry.pipelines.RegistryIngestionHistoryService;
-
-import java.util.UUID;
-
 import org.springframework.http.MediaType;
 import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 
-import io.swagger.v3.oas.annotations.Hidden;
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.UUID;
 
 @Hidden // TODO: Document?
 @RestController
@@ -44,15 +42,20 @@ public IngestionHistoryResource(RegistryIngestionHistoryService ingestionHistory
 
   @Override
   @GetMapping
-  public PagingResponse<IngestionProcess> history(Pageable pageable) {
-    return ingestionHistoryService.ingestionHistory(pageable);
+  public PagingResponse<IngestionProcess> history(
+      @RequestParam(value = "finishReason", required = false) @Nullable
+          List<FinishReason> finishReasons,
+      Pageable pageable) {
+    return ingestionHistoryService.ingestionHistory(finishReasons, pageable);
   }
 
   @Override
   @GetMapping("{datasetKey}")
   public PagingResponse<IngestionProcess> history(
-      @PathVariable("datasetKey") UUID datasetKey, Pageable pageable) {
-    return ingestionHistoryService.ingestionHistory(datasetKey, pageable);
+      @PathVariable("datasetKey") UUID datasetKey,
+      @RequestParam(value = "finishReason", required = false) @Nullable List<FinishReason> finishReasons,
+      Pageable pageable) {
+    return ingestionHistoryService.ingestionHistory(datasetKey, finishReasons, pageable);
   }
 
   @Override