feature/162-record-errors-on-feeds

.github/workflows/build.yml

@@ -4,7 +4,6 @@ on:
     branches:
       - master
       - develop
-      - 'feature/**'
   pull_request:
     types: [opened, synchronize, reopened]
 

.github/workflows/gitleaks.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - master
       - develop
-      - 'feature/**'
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -19,6 +18,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4.1.1
 
-      - uses: gitleaks/gitleaks-action@v2
+      - uses: gitleaks/gitleaks-action@v2.3.4
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITLEAKS_VERSION: latest

pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>3.2.3</version>
+        <version>3.2.4</version>
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
 
@@ -32,7 +32,7 @@
 
         <aalto-xml.version>1.3.2</aalto-xml.version>
         <findbugs.version>3.0.2</findbugs.version>
-        <html-sanitizer.version>20220608.1</html-sanitizer.version>
+        <html-sanitizer.version>20240325.1</html-sanitizer.version>
         <jetbrains.version>24.1.0</jetbrains.version>
         <json-unit.version>3.2.2</json-unit.version>
         <juery.version>1.2.4</juery.version>

sandside/src/main/java/fr/ght1pc9kc/baywatch/techwatch/domain/model/QueryContext.java → sandside/src/main/java/fr/ght1pc9kc/baywatch/common/domain/QueryContext.java

@@ -1,26 +1,25 @@
-package fr.ght1pc9kc.baywatch.techwatch.domain.model;
+package fr.ght1pc9kc.baywatch.common.domain;
 
 import fr.ght1pc9kc.juery.api.Criteria;
 import fr.ght1pc9kc.juery.api.PageRequest;
 import fr.ght1pc9kc.juery.api.Pagination;
 import lombok.Builder;
-import lombok.Value;
 import lombok.With;
 
 import java.util.List;
 
 import static fr.ght1pc9kc.baywatch.common.api.model.EntitiesProperties.ID;
 
-@Value
 @Builder
-public final class QueryContext {
-    @Builder.Default
-    public final Pagination pagination = Pagination.ALL;
-    public final Criteria filter;
-    @With
-    public final String userId;
-
-    public final List<String> teamMates;
+public record QueryContext(
+        Pagination pagination,
+        Criteria filter,
+        @With String userId,
+        List<String> teamMates
+) {
+    public QueryContext {
+        pagination = (pagination == null) ? Pagination.ALL : pagination;
+    }
 
     public static QueryContext from(PageRequest pr) {
         return new QueryContext(pr.pagination(), pr.filter(), null, List.of());

sandside/src/main/java/fr/ght1pc9kc/baywatch/common/infra/config/scalars/URIScalar.java

@@ -48,19 +48,12 @@ public URI serialize(@NotNull Object input,
             public @NotNull URI parseValue(@NotNull Object input,
                                            @NotNull GraphQLContext graphQLContext,
                                            @NotNull Locale locale) throws CoercingParseValueException {
-                String urlStr;
                 if (input instanceof String) {
-                    urlStr = String.valueOf(input);
+                    return parseURL(String.valueOf(input));
                 } else {
-                    Optional<URI> url = toURI(input);
-                    if (url.isEmpty()) {
-                        throw new CoercingParseValueException(
-                                "Expected a 'URI' like object but was '" + input.getClass().getSimpleName() + "'."
-                        );
-                    }
-                    return url.get();
+                    return toURI(input).orElseThrow(() -> new CoercingParseValueException(
+                            "Expected a 'URI' like object but was '" + input.getClass().getSimpleName() + "'."));
                 }
-                return parseURL(urlStr);
             }
 
             @Override
@@ -80,8 +73,10 @@ public URI serialize(@NotNull Object input,
             public @NotNull Value<StringValue> valueToLiteral(@NotNull Object input,
                                                               @NotNull GraphQLContext graphQLContext,
                                                               @NotNull Locale locale) {
-                URI url = serialize(input, graphQLContext, locale);
-                return StringValue.newStringValue(url.toString()).build();
+                return StringValue.newStringValue(
+                                serialize(input, graphQLContext, locale)
+                                        .toString())
+                        .build();
             }
 
             private URI parseURL(String input) {

sandside/src/main/java/fr/ght1pc9kc/baywatch/indexer/infra/handlers/IndexerEventHandler.java

@@ -0,0 +1,30 @@
+package fr.ght1pc9kc.baywatch.indexer.infra.handlers;
+
+import fr.ght1pc9kc.baywatch.indexer.api.FeedIndexerService;
+import fr.ght1pc9kc.baywatch.scraper.api.ScrapingEventHandler;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapResult;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingEventType;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Mono;
+
+import java.util.EnumSet;
+
+@Component
+@RequiredArgsConstructor
+@ConditionalOnProperty(name = "baywatch.indexer.enable", havingValue = "true")
+public final class IndexerEventHandler implements ScrapingEventHandler {
+
+    private final FeedIndexerService indexerService;
+
+    @Override
+    public Mono<Void> after(ScrapResult result) {
+        return indexerService.buildIndex();
+    }
+
+    @Override
+    public EnumSet<ScrapingEventType> eventTypes() {
+        return EnumSet.of(ScrapingEventType.FEED_SCRAPING);
+    }
+}

sandside/src/main/java/fr/ght1pc9kc/baywatch/opml/domain/OpmlServiceImpl.java

@@ -6,7 +6,7 @@
 import fr.ght1pc9kc.baywatch.security.api.model.User;
 import fr.ght1pc9kc.baywatch.security.domain.exceptions.UnauthenticatedUser;
 import fr.ght1pc9kc.baywatch.techwatch.api.model.WebFeed;
-import fr.ght1pc9kc.baywatch.techwatch.domain.model.QueryContext;
+import fr.ght1pc9kc.baywatch.common.domain.QueryContext;
 import fr.ght1pc9kc.baywatch.techwatch.infra.persistence.FeedRepository;
 import fr.ght1pc9kc.entity.api.Entity;
 import lombok.RequiredArgsConstructor;

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/api/ScrapingErrorsService.java

@@ -0,0 +1,16 @@
+package fr.ght1pc9kc.baywatch.scraper.api;
+
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingError;
+import fr.ght1pc9kc.entity.api.Entity;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+import java.util.Collection;
+
+public interface ScrapingErrorsService {
+    Flux<Entity<ScrapingError>> persist(Collection<Entity<ScrapingError>> errors);
+
+    Flux<Entity<ScrapingError>> list(Collection<String> feedsIds);
+
+    Mono<Void> purge(Collection<String> notInFeedsIds);
+}

sandside/src/main/java/fr/ght1pc9kc/baywatch/common/api/ScrapingEventHandler.java → sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/api/ScrapingEventHandler.java

@@ -1,9 +1,10 @@
-package fr.ght1pc9kc.baywatch.common.api;
+package fr.ght1pc9kc.baywatch.scraper.api;
 
 import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapResult;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingEventType;
 import reactor.core.publisher.Mono;
 
-import java.util.Set;
+import java.util.EnumSet;
 
 /**
  * The Scrapping Handlers allow inserting action before and after some type of events
@@ -44,5 +45,5 @@ default void onTerminate() {
      *
      * @return A set of event types
      */
-    Set<String> eventTypes();
+    EnumSet<ScrapingEventType> eventTypes();
 }

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/api/model/ScrapingError.java

@@ -0,0 +1,11 @@
+package fr.ght1pc9kc.baywatch.scraper.api.model;
+
+import java.time.Instant;
+
+public record ScrapingError(
+        Instant since,
+        Instant lastTime,
+        int status,
+        String label
+) {
+}

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/api/model/ScrapingEventType.java

@@ -0,0 +1,5 @@
+package fr.ght1pc9kc.baywatch.scraper.api.model;
+
+public enum ScrapingEventType {
+    FEED_SCRAPING;
+}

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/domain/FeedScraperServiceImpl.java

@@ -2,15 +2,16 @@
 
 import fr.ght1pc9kc.baywatch.common.api.HttpHeaders;
 import fr.ght1pc9kc.baywatch.common.api.HttpStatusCodes;
-import fr.ght1pc9kc.baywatch.common.api.ScrapingEventHandler;
 import fr.ght1pc9kc.baywatch.common.domain.DateUtils;
 import fr.ght1pc9kc.baywatch.common.domain.Try;
 import fr.ght1pc9kc.baywatch.scraper.api.FeedScraperPlugin;
 import fr.ght1pc9kc.baywatch.scraper.api.FeedScraperService;
 import fr.ght1pc9kc.baywatch.scraper.api.RssAtomParser;
 import fr.ght1pc9kc.baywatch.scraper.api.ScrapEnrichmentService;
+import fr.ght1pc9kc.baywatch.scraper.api.ScrapingEventHandler;
 import fr.ght1pc9kc.baywatch.scraper.api.model.AtomFeed;
 import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapResult;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingEventType;
 import fr.ght1pc9kc.baywatch.scraper.domain.model.ScrapedFeed;
 import fr.ght1pc9kc.baywatch.scraper.domain.model.ex.FeedScrapingException;
 import fr.ght1pc9kc.baywatch.scraper.domain.model.ex.ScrapingException;
@@ -82,7 +83,7 @@ public FeedScraperServiceImpl(Scheduler scraperScheduler,
         this.maintenancePersistencePort = maintenancePersistencePort;
         this.feedParser = feedParser;
         this.scrapingHandlers = scrapingHandlers.stream()
-                .filter(e -> e.eventTypes().contains("FEED_SCRAPING")).toList();
+                .filter(e -> e.eventTypes().contains(ScrapingEventType.FEED_SCRAPING)).toList();
         this.plugins = plugins;
         this.scrapEnrichmentService = scrapEnrichmentService;
         this.http = webClient;

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/domain/ScrapingErrorsServiceImpl.java

@@ -0,0 +1,45 @@
+package fr.ght1pc9kc.baywatch.scraper.domain;
+
+import fr.ght1pc9kc.baywatch.common.domain.QueryContext;
+import fr.ght1pc9kc.baywatch.scraper.api.ScrapingErrorsService;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingError;
+import fr.ght1pc9kc.baywatch.scraper.domain.ports.ScrapingAuthentFacade;
+import fr.ght1pc9kc.baywatch.scraper.domain.ports.ScrapingErrorPersistencePort;
+import fr.ght1pc9kc.entity.api.Entity;
+import fr.ght1pc9kc.juery.api.Criteria;
+import lombok.RequiredArgsConstructor;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+import java.util.Collection;
+
+import static fr.ght1pc9kc.baywatch.common.api.model.EntitiesProperties.ID;
+
+@RequiredArgsConstructor
+public class ScrapingErrorsServiceImpl implements ScrapingErrorsService {
+    private final ScrapingErrorPersistencePort persistencePort;
+    private final ScrapingAuthentFacade authentFacade;
+
+    @Override
+    public Flux<Entity<ScrapingError>> persist(Collection<Entity<ScrapingError>> errors) {
+        return authentFacade.getConnectedUser()
+                .filter(authentFacade::hasSystemRole)
+                .switchIfEmpty(Mono.error(() -> new IllegalAccessException("Persis scraping error not permitted !")))
+                .flatMapMany(u -> persistencePort.persist(errors));
+    }
+
+    @Override
+    public Flux<Entity<ScrapingError>> list(Collection<String> feedsIds) {
+        QueryContext query = QueryContext.all(Criteria.property(ID).in(feedsIds));
+        return persistencePort.list(query);
+    }
+
+    @Override
+    public Mono<Void> purge(Collection<String> notInFeedsIds) {
+        QueryContext query = QueryContext.all(Criteria.not(Criteria.property(ID).in(notInFeedsIds)));
+        return authentFacade.getConnectedUser()
+                .filter(authentFacade::hasSystemRole)
+                .switchIfEmpty(Mono.error(() -> new IllegalAccessException("Persis scraping error not permitted !")))
+                .flatMap(u -> persistencePort.delete(query));
+    }
+}

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/domain/actions/DeleteOrphanFeedHandler.java

@@ -1,6 +1,7 @@
 package fr.ght1pc9kc.baywatch.scraper.domain.actions;
 
-import fr.ght1pc9kc.baywatch.common.api.ScrapingEventHandler;
+import fr.ght1pc9kc.baywatch.scraper.api.ScrapingEventHandler;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingEventType;
 import fr.ght1pc9kc.baywatch.techwatch.api.SystemMaintenanceService;
 import fr.ght1pc9kc.baywatch.techwatch.api.model.News;
 import fr.ght1pc9kc.juery.api.Criteria;
@@ -10,7 +11,7 @@
 import reactor.core.publisher.Mono;
 import reactor.core.publisher.Sinks;
 
-import java.util.Set;
+import java.util.EnumSet;
 
 import static fr.ght1pc9kc.baywatch.common.api.model.EntitiesProperties.COUNT;
 import static fr.ght1pc9kc.baywatch.common.api.model.EntitiesProperties.FEED_ID;
@@ -60,8 +61,8 @@ public Mono<Void> before() {
     }
 
     @Override
-    public Set<String> eventTypes() {
-        return Set.of("FEED_SCRAPING");
+    public EnumSet<ScrapingEventType> eventTypes() {
+        return EnumSet.of(ScrapingEventType.FEED_SCRAPING);
     }
 
 }

sandside/src/main/java/fr/ght1pc9kc/baywatch/scraper/domain/actions/NewsUpdateNotificationHandler.java

@@ -1,14 +1,15 @@
 package fr.ght1pc9kc.baywatch.scraper.domain.actions;
 
-import fr.ght1pc9kc.baywatch.common.api.ScrapingEventHandler;
 import fr.ght1pc9kc.baywatch.notify.api.NotifyService;
 import fr.ght1pc9kc.baywatch.notify.api.model.EventType;
+import fr.ght1pc9kc.baywatch.scraper.api.ScrapingEventHandler;
 import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapResult;
+import fr.ght1pc9kc.baywatch.scraper.api.model.ScrapingEventType;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import reactor.core.publisher.Mono;
 
-import java.util.Set;
+import java.util.EnumSet;
 
 @Slf4j
 @RequiredArgsConstructor
@@ -26,7 +27,7 @@ public Mono<Void> after(ScrapResult result) {
     }
 
     @Override
-    public Set<String> eventTypes() {
-        return Set.of("FEED_SCRAPING");
+    public EnumSet<ScrapingEventType> eventTypes() {
+        return EnumSet.of(ScrapingEventType.FEED_SCRAPING);
     }
 }