From 36def9616e73fdac76e2a419d090d917e160d999 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:43:56 +0200 Subject: [PATCH 01/37] fix: broken compaction due to used uuids (#1373) * fix: broken compaction due to used uuids * feat: additional tests * feat: additional tests --- .../pagination/services/PageNumberParser.java | 23 ++++++++++ .../pagination/valueobjects/PageNumber.java | 42 ----------------- .../pagination/valueobjects/PartialUrl.java | 16 ++++--- .../pagenumber/NumericPageNumber.java | 37 +++++++++++++++ .../valueobjects/pagenumber/PageNumber.java | 9 ++++ .../pagenumber/UuidPageNumber.java | 46 +++++++++++++++++++ .../pagination/batch/PaginatorTest.java | 18 ++++---- .../services/PageNumberParserTest.java | 44 ++++++++++++++++++ .../valueobjects/PartialUrlTest.java | 42 +++++++++++++---- .../pagenumber/NumericPageNumberTest.java | 45 ++++++++++++++++++ .../pagenumber/UuidPageNumberTest.java | 45 ++++++++++++++++++ 11 files changed, 301 insertions(+), 66 deletions(-) create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParser.java delete mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageNumber.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumber.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/PageNumber.java create mode 100644 ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumber.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParserTest.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumberTest.java create mode 100644 ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumberTest.java diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParser.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParser.java new file mode 100644 index 000000000..a80806449 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParser.java @@ -0,0 +1,23 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.NumericPageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.UuidPageNumber; +import org.apache.commons.lang3.math.NumberUtils; + +import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber.PAGE_NUMBER_KEY; + +public class PageNumberParser { + public static PageNumber parse(String pageNumberString) { + if (!pageNumberString.contains(PAGE_NUMBER_KEY)) { + throw new IllegalArgumentException("Invalid page number: %s - Expected format: %s=".formatted(pageNumberString, PAGE_NUMBER_KEY)); + } + final String value = pageNumberString.replace(PAGE_NUMBER_KEY + "=", ""); + if(NumberUtils.isCreatable(value)) { + return new NumericPageNumber(Integer.parseInt(value)); + } + return UuidPageNumber.fromString(value); + } + + private PageNumberParser() {} +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageNumber.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageNumber.java deleted file mode 100644 index 1b3c1cd19..000000000 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PageNumber.java +++ /dev/null @@ -1,42 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; - -public class PageNumber { - public static final String PAGE_NUMBER = "pageNumber"; - private final int value; - - public PageNumber(int value) { - this.value = value; - } - - public String asString() { - return PAGE_NUMBER + "=" + value; - } - - public PageNumber increment() { - return new PageNumber(value + 1); - } - - public static PageNumber startPageNumber() { - return new PageNumber(1); - } - - public static PageNumber fromString(String pageNumberString) { - if (!pageNumberString.contains(PAGE_NUMBER)) { - throw new IllegalArgumentException("Invalid page number: %s - Expected format: %s=".formatted(pageNumberString, PAGE_NUMBER)); - } - return new PageNumber(Integer.parseInt(pageNumberString.replace(PAGE_NUMBER + "=", ""))); - } - - @Override - public final boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof PageNumber that)) return false; - - return value == that.value; - } - - @Override - public int hashCode() { - return value; - } -} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrl.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrl.java index 4db246bd5..8470aeaf6 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrl.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrl.java @@ -1,6 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; -import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageNumber.PAGE_NUMBER; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.services.PageNumberParser; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.NumericPageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber; + +import static be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber.PAGE_NUMBER_KEY; public class PartialUrl { private final String viewName; @@ -27,7 +31,7 @@ public PartialUrl createChild() { return new PartialUrl( viewName, bucketDescriptor, - pageNumber == null ? PageNumber.startPageNumber() : pageNumber.increment() + pageNumber == null ? NumericPageNumber.startPageNumber() : pageNumber.getNextPageNumber() ); } @@ -42,7 +46,7 @@ public static PartialUrl fromUrl(String url) { } if (isUrlOnlyPaged(parts[1])) { - return new PartialUrl(viewName, "", PageNumber.fromString(parts[1])); + return new PartialUrl(viewName, "", PageNumberParser.parse(parts[1])); } return createCompletePartialUrl(viewName, parts[1]); @@ -53,17 +57,17 @@ public boolean isNumberLess() { } private static boolean isUrlNumberless(String extendedDescriptor) { - return !extendedDescriptor.contains(PAGE_NUMBER); + return !extendedDescriptor.contains(PAGE_NUMBER_KEY); } private static boolean isUrlOnlyPaged(String extendedDescriptor) { - return extendedDescriptor.startsWith(PAGE_NUMBER); + return extendedDescriptor.startsWith(PAGE_NUMBER_KEY); } private static PartialUrl createCompletePartialUrl(String viewName, String extendedDescriptor) { final int lastIndex = extendedDescriptor.lastIndexOf("&"); final String bucketDescriptor = extendedDescriptor.substring(0, lastIndex); - final PageNumber pageNumber = PageNumber.fromString(extendedDescriptor.substring(lastIndex + 1)); + final PageNumber pageNumber = PageNumberParser.parse(extendedDescriptor.substring(lastIndex + 1)); return new PartialUrl(viewName, bucketDescriptor, pageNumber); } } diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumber.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumber.java new file mode 100644 index 000000000..5a5559cef --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumber.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber; + +public class NumericPageNumber implements PageNumber { + private final int value; + + public NumericPageNumber(int value) { + this.value = value; + } + + @Override + public String asString() { + return PAGE_NUMBER_KEY + "=" + value; + } + + @Override + public PageNumber getNextPageNumber() { + return new NumericPageNumber(value + 1); + } + + public static PageNumber startPageNumber() { + return new NumericPageNumber(1); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NumericPageNumber that)) return false; + return value == that.value; + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + value; + return result; + } +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/PageNumber.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/PageNumber.java new file mode 100644 index 000000000..6809268b3 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/PageNumber.java @@ -0,0 +1,9 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber; + +public interface PageNumber { + String PAGE_NUMBER_KEY = "pageNumber"; + + String asString(); + + PageNumber getNextPageNumber(); +} diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumber.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumber.java new file mode 100644 index 000000000..fa66a6c95 --- /dev/null +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumber.java @@ -0,0 +1,46 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber; + +import java.util.Objects; +import java.util.UUID; + +public class UuidPageNumber implements PageNumber { + private final String value; + + public UuidPageNumber(UUID uuid) { + this.value = uuid.toString(); + } + + public UuidPageNumber() { + this.value = UUID.randomUUID().toString(); + } + + public static UuidPageNumber fromString(String value) { + final UUID uuid = UUID.fromString(value); + return new UuidPageNumber(uuid); + } + + @Override + public String asString() { + return PAGE_NUMBER_KEY + "=" + value; + } + + @Override + public PageNumber getNextPageNumber() { + return new UuidPageNumber(); + } + + @Override + public final boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof UuidPageNumber that)) return false; + + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + Objects.hashCode(value); + return result; + } +} diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java index e07e51f0a..991650af8 100644 --- a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/batch/PaginatorTest.java @@ -3,7 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.NumericPageNumber; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.PartialUrl; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -53,15 +53,15 @@ void when_Paginating_withNoUnexpectedMembers_doNothing() { @Test void when_Paginating_withNumberedOpenPage_expectCorrectPages() { mockBucketId(); - Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(1)), PAGE_SIZE); when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); when(pageRepository.getOpenPage(BUCKET_ID)) .thenReturn(numberedPage); when(pageRepository.createNextPage(any())) - .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) - .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(3)), PAGE_SIZE)); paginator.execute(null, chunkContext); @@ -88,8 +88,8 @@ void when_Paginating_withUnNumberedOpenPage_expectCorrectPages() { when(pageRepository.createNextPage(any())) .thenReturn(numberedPage) - .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) - .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(3)), PAGE_SIZE)); paginator.execute(null, chunkContext); @@ -107,14 +107,14 @@ void when_Paginating_withUnNumberedOpenPage_expectCorrectPages() { @Test void when_Paginating_withSemiFilledNumberedOpenPage_expectCorrectPages() { mockBucketId(); - Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(1)), PAGE_SIZE,1); + Page numberedPage = new Page(1, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(1)), PAGE_SIZE,1); when(pageMemberRepository.getUnpaginatedMembersForBucket(BUCKET_ID)).thenReturn(List.of(1L, 2L, 3L, 4L, 5L)); when(pageRepository.getOpenPage(BUCKET_ID)) .thenReturn(numberedPage); when(pageRepository.createNextPage(any())) - .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(2)), PAGE_SIZE)) - .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new PageNumber(3)), PAGE_SIZE)); + .thenReturn(new Page(2, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(2)), PAGE_SIZE)) + .thenReturn(new Page(3, BUCKET_ID, new PartialUrl(VIEW_NAME, "", new NumericPageNumber(3)), PAGE_SIZE)); paginator.execute(null, chunkContext); diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParserTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParserTest.java new file mode 100644 index 000000000..79481703f --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/services/PageNumberParserTest.java @@ -0,0 +1,44 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.services; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.NumericPageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.UuidPageNumber; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PageNumberParserTest { + @Test + void test_CreateNumericPageNumber() { + final String pageNumberString = "pageNumber=2"; + + final PageNumber pageNumber = PageNumberParser.parse(pageNumberString); + + assertThat(pageNumber).isEqualTo(new NumericPageNumber(2)); + } + + @Test + void test_UuidNumericPageNumber() { + final String pageNumberString = "pageNumber=26929514-237c-11ed-861d-0242ac120002"; + + final PageNumber pageNumber = PageNumberParser.parse(pageNumberString); + + assertThat(pageNumber).isEqualTo(UuidPageNumber.fromString("26929514-237c-11ed-861d-0242ac120002")); + } + + @ParameterizedTest(name = "{0}") + @CsvSource(textBlock = """ + 2,Invalid page number: 2 - Expected format: pageNumber= + pageNumber=a,Invalid UUID string: a, + pageNumber=26929514-237c-11ed-861d,Invalid UUID string: 26929514-237c-11ed-861d + pageNumber=26929514-237c-11ed-861d-237c-11ed-861d,UUID string too large + """) + void test_InvalidCreation(String pageNumberString, String expectedErrorMessage) { + assertThatThrownBy(() -> PageNumberParser.parse(pageNumberString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedErrorMessage); + } +} \ No newline at end of file diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrlTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrlTest.java index fcd4046af..6f919747f 100644 --- a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrlTest.java +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/PartialUrlTest.java @@ -1,5 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.NumericPageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.PageNumber; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber.UuidPageNumber; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -36,28 +39,49 @@ void test_CreateChild(PartialUrl partialUrl, PartialUrl expectedChild) { .isEqualTo(expectedChild); } + @ParameterizedTest + @ArgumentsSource(NumberedChildPartialUrlProvider.class) + void test_CreatedNumberedChild(String partialUrl, Class expectedInstanceClass) { + final PartialUrl child = PartialUrl.fromUrl(partialUrl).createChild(); + + assertThat(child) + .extracting("pageNumber") + .isInstanceOf(expectedInstanceClass); + } + static class PartialUrlProvider implements ArgumentsProvider { @Override - public Stream provideArguments(ExtensionContext extensionContext) throws Exception { + public Stream provideArguments(ExtensionContext extensionContext) { return Stream.of( Arguments.of("/event-stream/paged", new PartialUrl("event-stream/paged", "", null)), Arguments.of("/event-stream/by-loc?tile=15/142/123", new PartialUrl("event-stream/by-loc", "tile=15/142/123", null)), Arguments.of("/event-stream/by-time?year=2024&month=06&day=28", new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", null)), - Arguments.of("/event-stream/paged?pageNumber=2", new PartialUrl("event-stream/paged", "", new PageNumber(2))), - Arguments.of("/event-stream/by-time?year=2024&month=06&day=28&pageNumber=2", new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new PageNumber(2))) + Arguments.of("/event-stream/paged?pageNumber=2", new PartialUrl("event-stream/paged", "", new NumericPageNumber(2))), + Arguments.of("/event-stream/by-time?year=2024&month=06&day=28&pageNumber=2", new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new NumericPageNumber(2))), + Arguments.of("/event-stream/by-time?year=2024&month=06&day=28&pageNumber=26929514-237c-11ed-861d-0242ac120002", new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", UuidPageNumber.fromString("26929514-237c-11ed-861d-0242ac120002"))) ); } } static class ChildPartialUrlProvider implements ArgumentsProvider { @Override - public Stream provideArguments(ExtensionContext extensionContext) throws Exception { + public Stream provideArguments(ExtensionContext extensionContext) { + return Stream.of( + Arguments.of(new PartialUrl("event-stream/paged", "", null), new PartialUrl("event-stream/paged", "", new NumericPageNumber(1))), + Arguments.of(new PartialUrl("event-stream/by-loc", "tile=15/142/123", null), new PartialUrl("event-stream/by-loc", "tile=15/142/123", new NumericPageNumber(1))), + Arguments.of(new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", null), new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new NumericPageNumber(1))), + Arguments.of(new PartialUrl("event-stream/paged", "", new NumericPageNumber(2)), new PartialUrl("event-stream/paged", "", new NumericPageNumber(3))), + Arguments.of(new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new NumericPageNumber(2)), new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new NumericPageNumber(3))) + ); + } + } + + static class NumberedChildPartialUrlProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext extensionContext) { return Stream.of( - Arguments.of(new PartialUrl("event-stream/paged", "", null), new PartialUrl("event-stream/paged", "", new PageNumber(1))), - Arguments.of(new PartialUrl("event-stream/by-loc", "tile=15/142/123", null), new PartialUrl("event-stream/by-loc", "tile=15/142/123", new PageNumber(1))), - Arguments.of(new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", null), new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new PageNumber(1))), - Arguments.of(new PartialUrl("event-stream/paged", "", new PageNumber(2)), new PartialUrl("event-stream/paged", "", new PageNumber(3))), - Arguments.of(new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new PageNumber(2)), new PartialUrl("event-stream/by-time", "year=2024&month=06&day=28", new PageNumber(3))) + Arguments.of("/event-stream/by-time?year=2024&month=06&day=28&pageNumber=2", NumericPageNumber.class), + Arguments.of("/event-stream/by-time?year=2024&month=06&day=28&pageNumber=26929514-237c-11ed-861d-0242ac120002", UuidPageNumber.class) ); } } diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumberTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumberTest.java new file mode 100644 index 000000000..d0d456fad --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/NumericPageNumberTest.java @@ -0,0 +1,45 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class NumericPageNumberTest { + private static final int PAGE_NUMBER_VALUE = 3; + private static final NumericPageNumber PAGE_NUMBER = new NumericPageNumber(PAGE_NUMBER_VALUE); + + @Test + void testEquality() { + final NumericPageNumber other = new NumericPageNumber(PAGE_NUMBER_VALUE); + + assertThat(PAGE_NUMBER) + .isEqualTo(other) + .isEqualTo(PAGE_NUMBER) + .hasSameHashCodeAs(other); + assertThat(other) + .isEqualTo(PAGE_NUMBER); + } + + @ParameterizedTest + @MethodSource + void testInequality(Object other) { + assertThat(other).isNotEqualTo(PAGE_NUMBER); + + if(other != null) { + assertThat(other).doesNotHaveSameHashCodeAs(PAGE_NUMBER); + } + } + + static Stream testInequality() { + return Stream.of( + new NumericPageNumber(10), + PAGE_NUMBER_VALUE, + new UuidPageNumber(), + null + ); + } +} \ No newline at end of file diff --git a/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumberTest.java b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumberTest.java new file mode 100644 index 000000000..bf5e8120d --- /dev/null +++ b/ldes-server-pagination/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/valueobjects/pagenumber/UuidPageNumberTest.java @@ -0,0 +1,45 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.valueobjects.pagenumber; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class UuidPageNumberTest { + private static final String UUID = "26929514-237c-11ed-861d-0242ac120002"; + private static final UuidPageNumber PAGE_NUMBER = UuidPageNumber.fromString(UUID); + + @Test + void testEquality() { + final UuidPageNumber other = UuidPageNumber.fromString(UUID); + + assertThat(PAGE_NUMBER) + .isEqualTo(other) + .isEqualTo(PAGE_NUMBER) + .hasSameHashCodeAs(other); + assertThat(other) + .isEqualTo(PAGE_NUMBER); + } + + @ParameterizedTest + @MethodSource + void testInequality(Object other) { + assertThat(other).isNotEqualTo(PAGE_NUMBER); + + if(other != null) { + assertThat(other).doesNotHaveSameHashCodeAs(PAGE_NUMBER); + } + } + + static Stream testInequality() { + return Stream.of( + UuidPageNumber.fromString("fe15639d-526a-42e2-8d70-cbc239d7156c"), + UUID, + new NumericPageNumber(1), + null + ); + } +} \ No newline at end of file From dba8d86a7febcdae704dd9b191733bc9aa1a2909 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:45:07 +0200 Subject: [PATCH 02/37] ci: prepare release 3.5.0 (#1376) --- ldes-fragmentisers/ldes-fragmentisers-common/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml | 2 +- .../ldes-fragmentisers-timebased-hierarchical/pom.xml | 2 +- ldes-fragmentisers/pom.xml | 2 +- ldes-server-admin/pom.xml | 2 +- ldes-server-application/pom.xml | 2 +- ldes-server-compaction/pom.xml | 2 +- ldes-server-domain/pom.xml | 2 +- ldes-server-infra-postgres/pom.xml | 2 +- ldes-server-infra-postgres/postgres-admin-repository/pom.xml | 2 +- .../postgres-fragmentation-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-ingest-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-liquibase/pom.xml | 2 +- .../postgres-pagination-repository/pom.xml | 2 +- .../postgres-retention-repository/pom.xml | 2 +- ldes-server-instrumentation/pom.xml | 2 +- ldes-server-integration-test/pom.xml | 2 +- ldes-server-pagination/pom.xml | 2 +- ldes-server-port-fetch-rest/pom.xml | 2 +- ldes-server-port-fetch/pom.xml | 2 +- ldes-server-port-ingest-rest/pom.xml | 2 +- ldes-server-port-ingest/pom.xml | 2 +- ldes-server-retention/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index 496386d18..fbca78b93 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml index b09b62ffa..47cf181a3 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml index 1cb9db965..8d18b7bc3 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT 4.0.0 jar diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml index 837dc6fa6..1b440360e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-fragmentisers-timebased-hierarchical diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index e60483037..bf23eb440 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-fragmentisers diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 47fecf78a..505bcfd1c 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 0bce3852d..ddf55b6ca 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-application diff --git a/ldes-server-compaction/pom.xml b/ldes-server-compaction/pom.xml index ed5971a3a..8c75d3c72 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-compaction/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-compaction diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index 90cc3253e..91adc29ce 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-domain diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 1bd4c3efc..2cc0521c7 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT pom diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index af0cdc88e..da8b7d552 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index 10572b603..96669419c 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index 234333ebc..0bdfb7620 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml index f2c0848f6..587cae58f 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/pom.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-liquibase diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index 4774ad589..c4771fa76 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-pagination-repository diff --git a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml index 403e150fd..95c2d1f45 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT postgres-retention-repository diff --git a/ldes-server-instrumentation/pom.xml b/ldes-server-instrumentation/pom.xml index b2b1f472a..6c73a9048 100644 --- a/ldes-server-instrumentation/pom.xml +++ b/ldes-server-instrumentation/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-instrumentation diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index 2ed29db81..0ac0e23e1 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-integration-test diff --git a/ldes-server-pagination/pom.xml b/ldes-server-pagination/pom.xml index b92029b8d..3b351eebe 100644 --- a/ldes-server-pagination/pom.xml +++ b/ldes-server-pagination/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-pagination diff --git a/ldes-server-port-fetch-rest/pom.xml b/ldes-server-port-fetch-rest/pom.xml index 109eec0d7..16e3325bc 100644 --- a/ldes-server-port-fetch-rest/pom.xml +++ b/ldes-server-port-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-port-fetch-rest diff --git a/ldes-server-port-fetch/pom.xml b/ldes-server-port-fetch/pom.xml index 732383753..a3973f401 100644 --- a/ldes-server-port-fetch/pom.xml +++ b/ldes-server-port-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-port-fetch diff --git a/ldes-server-port-ingest-rest/pom.xml b/ldes-server-port-ingest-rest/pom.xml index 3b2d318eb..5298c7bf2 100644 --- a/ldes-server-port-ingest-rest/pom.xml +++ b/ldes-server-port-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-port-ingest-rest diff --git a/ldes-server-port-ingest/pom.xml b/ldes-server-port-ingest/pom.xml index 55ef8001b..0f5b459c5 100644 --- a/ldes-server-port-ingest/pom.xml +++ b/ldes-server-port-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-port-ingest diff --git a/ldes-server-retention/pom.xml b/ldes-server-retention/pom.xml index f2dddcdbb..ed39afd8e 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-retention/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT ldes-server-retention diff --git a/pom.xml b/pom.xml index e079614b1..7b91450a0 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0-SNAPSHOT + 3.5.0-SNAPSHOT pom From 8c1e1f850079f424cec2b591b0c381657e5d8ba4 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:36:56 +0200 Subject: [PATCH 03/37] chore: restructuring to maintenance (#1375) * chore: restructure to a common maintenance module * chore: restructure to a common maintenance module * chore: clearance of todos * chore: update dockerfile * chore: resolve rebase conflicts * chore: restructure to a common maintenance module * chore: clearance of todos * chore: update dockerfile --- .github/Dockerfile | 5 +- ldes-server-application/pom.xml | 12 +-- .../server/domain/constants/ServerConfig.java | 1 + .../ldes-server-compaction}/pom.xml | 3 +- .../eventhandlers/ViewCapacityCreator.java | 0 .../eventhandlers/ViewCapacityDeleter.java | 0 .../services/CompactionCandidateService.java | 0 .../services/CompactionCandidateSorter.java | 0 .../services/CompactionScheduler.java | 0 .../services/FragmentDeletionScheduler.java | 0 .../FragmentationConfigCompaction.java | 0 .../services/PageDeletionTimeSetter.java | 0 .../services/PaginationCompactionService.java | 0 .../services/SchedulingConfigCompaction.java | 0 .../entities/CompactedFragmentCreator.java | 0 .../domain/entities/ViewCapacity.java | 0 .../domain/repository/ViewCollection.java | 0 .../domain/repository/ViewCollectionImpl.java | 0 .../CompactionCandidateServiceTest.java | 0 .../services/CompactionSchedulerTest.java | 0 .../FragmentDeletionSchedulerTest.java | 0 .../services/PageDeletionTimeSetterTest.java | 0 .../PaginationCompactionServiceTest.java | 0 .../repository/ViewCollectionImplTest.java | 0 .../ldes-server-maintenance-common/pom.xml | 24 +++++ .../ldes-server-retention}/pom.xml | 2 +- .../server/retention/config/AsyncConfig.java | 0 .../retention/entities/MemberProperties.java | 0 .../DeletionPolicyCollection.java | 0 .../DeletionPolicyCollectionImpl.java | 0 .../MemberPropertiesRepository.java | 0 .../repositories/PageMemberRepository.java | 0 .../RetentionPolicyCollection.java | 0 .../RetentionPolicyCollectionImpl.java | 0 .../creation/RetentionPolicyConstants.java | 0 .../creation/RetentionPolicyCreator.java | 0 .../creation/RetentionPolicyFactory.java | 0 .../creation/RetentionPolicyFactoryImpl.java | 0 .../creation/timebased/DurationParser.java | 0 .../timebased/DurationParserException.java | 0 .../TimeBasedRetentionPolicyCreator.java | 0 .../VersionBasedRetentionPolicyCreator.java | 0 .../definition/RetentionPolicy.java | 0 .../definition/RetentionPolicyType.java | 0 .../TimeAndVersionBasedRetentionPolicy.java | 0 .../timebased/TimeBasedRetentionPolicy.java | 0 .../VersionBasedRetentionPolicy.java | 0 .../execution/MemberRemover.java | 0 .../execution/MemberRemoverImpl.java | 0 .../execution/RetentionService.java | 0 .../execution/SchedulingConfig.java | 0 .../spi/RetentionPolicyEmptinessChecker.java | 0 .../valueobjects/EventStreamProperties.java | 0 .../src/main/java/module-info.java | 0 .../DeletionPolicyCollectionImplTest.java | 0 .../RetentionPolicyCollectionImplTest.java | 0 .../RetentionPolicyFactoryImplTest.java | 0 .../timebased/DurationParserTest.java | 0 .../TimeBasedRetentionPolicyCreatorTest.java | 0 ...ersionBasedRetentionPolicyCreatorTest.java | 0 ...imeAndVersionBasedRetentionPolicyTest.java | 0 .../execution/MemberRemoverImplTest.java | 0 .../execution/RetentionServiceTest.java | 0 .../EventStreamPropertiesTest.java | 4 +- .../retentionpolicy-with-unknown-type.ttl | 0 .../retentionpolicy-without-type.ttl | 0 .../valid_timeandversionbased_it.ttl | 0 .../timebased/invalid_timebased.ttl | 0 .../timebased/valid_timebased.ttl | 0 .../timebased/valid_timebased_it.ttl | 0 .../versionbased/invalid_versionbased.ttl | 0 .../versionbased/valid_versionbased.ttl | 0 .../versionbased/valid_versionbased_it.ttl | 0 ldes-server-maintenance/pom.xml | 21 +++++ .../repositories/EventStreamCollection.java | 7 -- .../EventStreamCollectionImpl.java | 40 --------- .../repositories/ViewCollection.java | 41 --------- .../EventStreamCollectionImplTest.java | 45 ---------- .../repositories/ViewCollectionTest.java | 90 ------------------- pom.xml | 3 +- 80 files changed, 61 insertions(+), 237 deletions(-) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/pom.xml (98%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityCreator.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityDeleter.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetter.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/CompactedFragmentCreator.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/ViewCapacity.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollection.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImpl.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetterTest.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java (100%) rename {ldes-server-compaction => ldes-server-maintenance/ldes-server-compaction}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImplTest.java (100%) create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/pom.xml rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/pom.xml (97%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberProperties.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/PageMemberRepository.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyConstants.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyCreator.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactory.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImpl.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParser.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserException.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreator.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreator.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicyType.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicy.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timebased/TimeBasedRetentionPolicy.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/versionbased/VersionBasedRetentionPolicy.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemover.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImpl.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamProperties.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/main/java/module-info.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImplTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreatorTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreatorTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicyTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImplTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java (94%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/retentionpolicy-with-unknown-type.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/retentionpolicy-without-type.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/timeandversionbased/valid_timeandversionbased_it.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/timebased/invalid_timebased.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/timebased/valid_timebased.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/timebased/valid_timebased_it.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/versionbased/invalid_versionbased.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/versionbased/valid_versionbased.ttl (100%) rename {ldes-server-retention => ldes-server-maintenance/ldes-server-retention}/src/test/resources/retentionpolicy/versionbased/valid_versionbased_it.ttl (100%) create mode 100644 ldes-server-maintenance/pom.xml delete mode 100644 ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollection.java delete mode 100644 ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImpl.java delete mode 100644 ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollection.java delete mode 100644 ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImplTest.java delete mode 100644 ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollectionTest.java diff --git a/.github/Dockerfile b/.github/Dockerfile index 3a1907b8e..d6f910a03 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -21,8 +21,9 @@ COPY ./ldes-fragmentisers/ldes-fragmentisers-geospatial/target/ldes-fragmentiser COPY ./ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/target/ldes-fragmentisers-timebased-hierarchical-jar-with-dependencies.jar ./lib/ COPY ./ldes-fragmentisers/ldes-fragmentisers-reference/target/ldes-fragmentisers-reference-jar-with-dependencies.jar ./lib/ COPY ./ldes-server-pagination/target/ldes-server-pagination-jar-with-dependencies.jar ./lib/ -COPY ./ldes-server-retention/target/ldes-server-retention-jar-with-dependencies.jar ./lib/ -COPY ./ldes-server-compaction/target/ldes-server-compaction-jar-with-dependencies.jar ./lib/ +COPY ./ldes-server-maintenance/ldes-server-maintenance-common/target/ldes-server-maintenance-common-jar-with-dependencies.jar ./lib/ +COPY ./ldes-server-maintenance/ldes-server-retention/target/ldes-server-retention-jar-with-dependencies.jar ./lib/ +COPY ./ldes-server-maintenance/ldes-server-compaction/target/ldes-server-compaction-jar-with-dependencies.jar ./lib/ COPY ./ldes-server-instrumentation/target/ldes-server-instrumentation-jar-with-dependencies.jar ./lib/ ## Dependency for pyroscope diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index ddf55b6ca..d0c18514e 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -190,8 +190,13 @@ - retention + maintenance + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-maintenance-common + ${project.version} + be.vlaanderen.informatievlaanderen.vsds ldes-server-retention @@ -202,11 +207,6 @@ postgres-retention-repository ${project.version} - - - - compaction - be.vlaanderen.informatievlaanderen.vsds ldes-server-compaction diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java index 524637185..f4ca8fdea 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java @@ -19,6 +19,7 @@ public class ServerConfig { public static final String USE_RELATIVE_URL_KEY = "${ldes-server.use-relative-url:" + DEFAULT_USE_RELATIVE_URL + "}"; public static final String MAX_JSONLD_CACHE_CAPACITY = "${ldes-server.max-jsonld-cache-capacity:" + DEFAULT_MAX_JSONLD_CACHE_CAPACITY + "}"; + private String hostName; private String compactionDuration; private String retentionCron; diff --git a/ldes-server-compaction/pom.xml b/ldes-server-maintenance/ldes-server-compaction/pom.xml similarity index 98% rename from ldes-server-compaction/pom.xml rename to ldes-server-maintenance/ldes-server-compaction/pom.xml index 8c75d3c72..dfe82044f 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-maintenance/ldes-server-compaction/pom.xml @@ -5,11 +5,12 @@ 4.0.0 be.vlaanderen.informatievlaanderen.vsds - ldes-server + ldes-server-maintenance 3.5.0-SNAPSHOT ldes-server-compaction + jar diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityCreator.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityCreator.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityCreator.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityCreator.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityDeleter.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityDeleter.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityDeleter.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/eventhandlers/ViewCapacityDeleter.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetter.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetter.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetter.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetter.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/CompactedFragmentCreator.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/CompactedFragmentCreator.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/CompactedFragmentCreator.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/CompactedFragmentCreator.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/ViewCapacity.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/ViewCapacity.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/ViewCapacity.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/entities/ViewCapacity.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollection.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollection.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollection.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollection.java diff --git a/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImpl.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImpl.java similarity index 100% rename from ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImpl.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImpl.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetterTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetterTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetterTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PageDeletionTimeSetterTest.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java diff --git a/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImplTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImplTest.java similarity index 100% rename from ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImplTest.java rename to ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/domain/repository/ViewCollectionImplTest.java diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml b/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml new file mode 100644 index 000000000..7d0336c06 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-maintenance + 3.5.0-SNAPSHOT + + + ldes-server-maintenance-common + + + ${project.artifactId} + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + \ No newline at end of file diff --git a/ldes-server-retention/pom.xml b/ldes-server-maintenance/ldes-server-retention/pom.xml similarity index 97% rename from ldes-server-retention/pom.xml rename to ldes-server-maintenance/ldes-server-retention/pom.xml index ed39afd8e..74725e7c3 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-maintenance/ldes-server-retention/pom.xml @@ -5,7 +5,7 @@ 4.0.0 be.vlaanderen.informatievlaanderen.vsds - ldes-server + ldes-server-maintenance 3.5.0-SNAPSHOT diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberProperties.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberProperties.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberProperties.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/MemberProperties.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/PageMemberRepository.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/PageMemberRepository.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/PageMemberRepository.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/PageMemberRepository.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyConstants.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyConstants.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyConstants.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyConstants.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyCreator.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyCreator.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyCreator.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyCreator.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactory.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactory.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactory.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactory.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImpl.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImpl.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImpl.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParser.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParser.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParser.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParser.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserException.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserException.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserException.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserException.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreator.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreator.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreator.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreator.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreator.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreator.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreator.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreator.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicyType.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicyType.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicyType.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicyType.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicy.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicy.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicy.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicy.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timebased/TimeBasedRetentionPolicy.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timebased/TimeBasedRetentionPolicy.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timebased/TimeBasedRetentionPolicy.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timebased/TimeBasedRetentionPolicy.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/versionbased/VersionBasedRetentionPolicy.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/versionbased/VersionBasedRetentionPolicy.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/versionbased/VersionBasedRetentionPolicy.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/versionbased/VersionBasedRetentionPolicy.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemover.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemover.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemover.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemover.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImpl.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImpl.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImpl.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamProperties.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamProperties.java similarity index 100% rename from ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamProperties.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamProperties.java diff --git a/ldes-server-retention/src/main/java/module-info.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java similarity index 100% rename from ldes-server-retention/src/main/java/module-info.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImplTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImplTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/RetentionPolicyFactoryImplTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/DurationParserTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreatorTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreatorTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreatorTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/timebased/TimeBasedRetentionPolicyCreatorTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreatorTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreatorTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreatorTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/creation/versionbased/VersionBasedRetentionPolicyCreatorTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicyTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicyTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicyTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/timeandversionbased/TimeAndVersionBasedRetentionPolicyTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImplTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImplTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/MemberRemoverImplTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java similarity index 100% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java similarity index 94% rename from ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java rename to ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java index 007e200b7..270481760 100644 --- a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/valueobjects/EventStreamPropertiesTest.java @@ -14,8 +14,8 @@ class EventStreamPropertiesTest { - private final static String TIMESTAMP_PATH = "http://www.w3.org/ns/prov#generatedAtTime"; - private final static String VERSION_OF_PATH = "http://purl.org/dc/terms/isVersionOf"; + private static final String TIMESTAMP_PATH = "http://www.w3.org/ns/prov#generatedAtTime"; + private static final String VERSION_OF_PATH = "http://purl.org/dc/terms/isVersionOf"; @Test void test_equality() { diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-with-unknown-type.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-with-unknown-type.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-with-unknown-type.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-with-unknown-type.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-without-type.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-without-type.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-without-type.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/retentionpolicy-without-type.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/timeandversionbased/valid_timeandversionbased_it.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timeandversionbased/valid_timeandversionbased_it.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/timeandversionbased/valid_timeandversionbased_it.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timeandversionbased/valid_timeandversionbased_it.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/timebased/invalid_timebased.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/invalid_timebased.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/timebased/invalid_timebased.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/invalid_timebased.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased_it.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased_it.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased_it.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/timebased/valid_timebased_it.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/invalid_versionbased.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/invalid_versionbased.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/versionbased/invalid_versionbased.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/invalid_versionbased.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased.ttl diff --git a/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased_it.ttl b/ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased_it.ttl similarity index 100% rename from ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased_it.ttl rename to ldes-server-maintenance/ldes-server-retention/src/test/resources/retentionpolicy/versionbased/valid_versionbased_it.ttl diff --git a/ldes-server-maintenance/pom.xml b/ldes-server-maintenance/pom.xml new file mode 100644 index 000000000..cf4e053c8 --- /dev/null +++ b/ldes-server-maintenance/pom.xml @@ -0,0 +1,21 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.vsds + ldes-server + 3.5.0-SNAPSHOT + + + ldes-server-maintenance + pom + + + ldes-server-maintenance-common + ldes-server-retention + ldes-server-compaction + + + \ No newline at end of file diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollection.java b/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollection.java deleted file mode 100644 index 3ebaf58a4..000000000 --- a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollection.java +++ /dev/null @@ -1,7 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.retention.valueobjects.EventStreamProperties; - -public interface EventStreamCollection { - EventStreamProperties getEventStreamProperties(String collectionName); -} diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImpl.java b/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImpl.java deleted file mode 100644 index 7012c1929..000000000 --- a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.valueobjects.EventStreamProperties; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.Map; - -@Component -public class EventStreamCollectionImpl implements EventStreamCollection { - - private final Map eventStreamMap; - - public EventStreamCollectionImpl() { - this.eventStreamMap = new HashMap<>(); - } - - public EventStreamProperties getEventStreamProperties(String collectionName) { - if (eventStreamMap.containsKey(collectionName)) { - return eventStreamMap.get(collectionName); - } - throw new MissingResourceException("eventstream", collectionName); - } - - @EventListener - public void handleEventStreamCreatedEvent(EventStreamCreatedEvent event) { - eventStreamMap.put(event.eventStream().getCollection(), - new EventStreamProperties(event.eventStream().getVersionOfPath(), - event.eventStream().getTimestampPath())); - } - - @EventListener - public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { - eventStreamMap.remove(event.collectionName()); - } -} diff --git a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollection.java b/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollection.java deleted file mode 100644 index 7bf8f0167..000000000 --- a/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollection.java +++ /dev/null @@ -1,41 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.*; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -@Component -public class ViewCollection { - - private final Map views = new HashMap<>(); - - @EventListener({ViewAddedEvent.class, ViewInitializationEvent.class}) - public void handle(ViewSupplier event) { - views.put(event.viewSpecification().getName(), event.viewSpecification()); - } - - @EventListener - public void handle(ViewDeletedEvent event) { - views.remove(event.getViewName()); - } - - @EventListener - public void handle(EventStreamDeletedEvent event) { - views.keySet().stream() - .filter(viewName -> viewName.getCollectionName().equals(event.collectionName())) - .toList() - .forEach(views::remove); - - } - - public Collection getViews() { - return views.values(); - } - -} diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImplTest.java b/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImplTest.java deleted file mode 100644 index 6fb7a3c44..000000000 --- a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventStreamCollectionImplTest.java +++ /dev/null @@ -1,45 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.valueobjects.EventStreamProperties; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class EventStreamCollectionImplTest { - - private final static String COLLECTION = "COLLECTION"; - private final static String TIMESTAMP_PATH = "http://www.w3.org/ns/prov#generatedAtTime"; - private final static String VERSION_OF_PATH = "http://purl.org/dc/terms/isVersionOf"; - private final static boolean VERSION_CREATION_ENABLED = false; - private final EventStreamCollectionImpl eventStreamCollection = new EventStreamCollectionImpl(); - - @Test - void when_EventStreamDoesNotExist_then_MissingEventStreamExceptionIsThrown() { - eventStreamCollection.handleEventStreamCreatedEvent( - new EventStreamCreatedEvent(new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED))); - eventStreamCollection.handleEventStreamDeletedEvent(new EventStreamDeletedEvent(COLLECTION)); - - assertThatThrownBy(() -> eventStreamCollection.getEventStreamProperties(COLLECTION)) - .isInstanceOf(MissingResourceException.class) - .hasMessage("Resource of type: eventstream with id: %s could not be found.", COLLECTION); - } - - @Test - void when_EventStreamExists_then_EventStreamPropertiesCanBeRetrieved() { - eventStreamCollection.handleEventStreamCreatedEvent( - new EventStreamCreatedEvent(new EventStream(COLLECTION, TIMESTAMP_PATH, VERSION_OF_PATH, VERSION_CREATION_ENABLED))); - - EventStreamProperties actualEventStreamProperties = eventStreamCollection.getEventStreamProperties(COLLECTION); - - EventStreamProperties expectedEventStreamProperties = new EventStreamProperties(VERSION_OF_PATH, - TIMESTAMP_PATH); - - assertThat(actualEventStreamProperties).isEqualTo(expectedEventStreamProperties); - } - -} diff --git a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollectionTest.java b/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollectionTest.java deleted file mode 100644 index 2c2cdb7e9..000000000 --- a/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewCollectionTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.*; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; - -import java.util.List; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; - -class ViewCollectionTest { - - private ViewCollection viewCollection; - - @BeforeEach - void setUp() { - viewCollection = new ViewCollection(); - } - - @ParameterizedTest - @ArgumentsSource(ViewSupplierProvider.class) - void testHandleViewAddedEvent(ViewSupplier viewSupplier) { - viewCollection.handle(viewSupplier); - - assertThat(viewCollection.getViews()).contains(viewSupplier.viewSpecification()); - } - - static class ViewSupplierProvider implements ArgumentsProvider { - - ViewName viewName = ViewName.fromString("col/view"); - ViewSpecification viewSpecification = new ViewSpecification(viewName, List.of(), List.of(), 10); - - @Override - public Stream provideArguments(ExtensionContext context) { - return Stream.of( - Arguments.of(new ViewInitializationEvent(viewSpecification)), - Arguments.of(new ViewAddedEvent(viewSpecification))); - } - - } - - @Test - void testHandleViewDeletedEvent() { - final ViewName viewNameA = ViewName.fromString("col/viewA"); - final ViewSpecification viewSpecificationA = new ViewSpecification(viewNameA, List.of(), List.of(), 10); - final ViewName viewNameB = ViewName.fromString("col/viewB"); - final ViewSpecification viewSpecificationB = new ViewSpecification(viewNameB, List.of(), List.of(), 10); - - viewCollection.handle(new ViewAddedEvent(viewSpecificationA)); - viewCollection.handle(new ViewAddedEvent(viewSpecificationB)); - - assertThat(viewCollection.getViews()).hasSize(2); - - viewCollection.handle(new ViewDeletedEvent(viewNameA)); - - assertThat(viewCollection.getViews()).hasSize(1); - assertThat(viewCollection.getViews()).contains(viewSpecificationB); - } - - @Test - void testHandleEventStreamDeletedEvent() { - final String collectionNameA = "colA"; - final ViewName viewNameAA = new ViewName(collectionNameA, "viewA"); - final ViewSpecification viewSpecificationA = new ViewSpecification(viewNameAA, List.of(), List.of(), 10); - final ViewName viewNameAB = new ViewName(collectionNameA, "viewB"); - final ViewSpecification viewSpecificationB = new ViewSpecification(viewNameAB, List.of(), List.of(), 10); - final ViewName viewNameBC = ViewName.fromString("colB/viewC"); - final ViewSpecification viewSpecificationC = new ViewSpecification(viewNameBC, List.of(), List.of(), 10); - - viewCollection.handle(new ViewAddedEvent(viewSpecificationA)); - viewCollection.handle(new ViewAddedEvent(viewSpecificationB)); - viewCollection.handle(new ViewAddedEvent(viewSpecificationC)); - - assertThat(viewCollection.getViews()).hasSize(3); - - viewCollection.handle(new EventStreamDeletedEvent(collectionNameA)); - - assertThat(viewCollection.getViews()).hasSize(1); - assertThat(viewCollection.getViews()).contains(viewSpecificationC); - } - -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7b91450a0..a9f1f0da0 100644 --- a/pom.xml +++ b/pom.xml @@ -20,13 +20,12 @@ ldes-server-admin ldes-server-port-ingest ldes-server-port-ingest-rest - ldes-server-retention ldes-server-port-fetch ldes-server-port-fetch-rest - ldes-server-compaction ldes-server-integration-test ldes-server-instrumentation ldes-server-pagination + ldes-server-maintenance From a9b5df8071efc22dd8518d11f3cf4b5c939ce17d Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:52:24 +0200 Subject: [PATCH 04/37] feat: simplify maintenance (#1374) * feat: old service converted into two tasklets * feat: batch job added * feat: partitioning added to retention steps * chore: better naming * feat: additional tests * feat: all maintenance moved to spring batch * fix: broken test * feat: conditional running retention jobs * chore: maintenancejob should not be a bean * fix: broken test * feat: increased coverage * feat: retire old crons and add new single cron * fix: broken test due to invalid property config * feat: PR feedback + rework CompactionCandidateSorter * chore: remove redundant method --- docs/how-to-run.md | 18 +- .../fragmentation/FragmentationService.java | 4 +- .../batch/BatchConfiguration.java | 1 + .../server/domain/constants/ServerConfig.java | 33 ++-- .../domain/constants/ServerConfigTest.java | 16 +- .../postgres/PagePostgresRepository.java | 12 +- .../mapper/CompactionCandidateMapper.java | 17 ++ .../CompactionCandidateProjection.java | 4 - .../repository/PageEntityRepository.java | 14 +- .../server/LdesServerIntegrationTest.java | 12 +- .../resources/application-postgres-test.yml | 3 +- .../features/retention/retention.feature | 14 +- .../services/CompactionCandidateService.java | 49 ------ .../services/CompactionCandidateSorter.java | 92 +++++----- .../services/CompactionScheduler.java | 52 ------ .../services/FragmentDeletionScheduler.java | 34 ---- .../FragmentationConfigCompaction.java | 15 -- .../services/SchedulingConfigCompaction.java | 10 -- .../valueobjects/CompactedPages.java | 34 ++++ .../valueobjects/CompactionCandidates.java | 34 ++++ .../valueobjects/CompactionPageCapacity.java | 22 +++ .../batch/CompactionStepDefinitions.java | 42 +++++ .../compaction/batch/CompactionTask.java | 43 +++++ .../CompactionWriter.java} | 28 +-- .../compaction/batch/PageDeletionTask.java | 24 +++ .../batch/ViewCapacityPartitioner.java | 30 ++++ .../CompactionCandidateServiceTest.java | 69 -------- .../CompactionCandidateSorterTest.java | 34 ++++ .../services/CompactionSchedulerTest.java | 50 ------ .../FragmentDeletionSchedulerTest.java | 43 ----- .../PaginationCompactionServiceTest.java | 56 ------ .../compaction/batch/CompactionTaskTest.java | 67 ++++++++ .../batch/CompactionWriterTest.java | 59 +++++++ .../batch/PageDeletionTaskTest.java | 26 +++ .../batch/ViewCapacityPartitionerTest.java | 56 ++++++ .../ldes-server-maintenance-common/pom.xml | 34 ++++ .../maintenance/batch/MaintenanceFlows.java | 50 ++++++ .../maintenance/batch/MaintenanceService.java | 50 ++++++ .../exceptions/MaintenanceJobException.java | 9 + .../RetentionPolicyEmptinessChecker.java | 5 + .../DecidedFlowExecutionStatus.java | 24 +++ .../src/main/java/module-info.java | 8 + .../batch/MaintenanceJobDefinitionTest.java | 161 ++++++++++++++++++ .../batch/MaintenanceServiceTest.java | 83 +++++++++ .../ldes-server-retention/pom.xml | 4 + .../decider/RetentionExecutionDecider.java | 25 +++ .../decider/RetentionExecutionDeciders.java | 20 +++ .../RetentionPartitionerConfig.java | 20 +++ .../RetentionPolicyPartitioner.java | 30 ++++ .../EventSourceRetentionTask.java | 46 +++++ .../batch/retentiontasklet/RetentionTask.java | 23 +++ .../retentiontasklet/ViewRetentionTask.java | 42 +++++ .../RetentionStepDefinitions.java | 65 +++++++ .../server/retention/config/AsyncConfig.java | 9 - .../EventSourceRetentionPolicyProvider.java | 28 +++ .../RetentionPolicyProvider.java} | 9 +- .../entities/ViewRetentionPolicyProvider.java | 28 +++ .../DeletionPolicyCollectionImpl.java | 52 ------ .../RetentionPolicyCollection.java | 12 -- .../RetentionPolicyCollectionImpl.java | 67 -------- .../EventSourceRetentionPolicyCollection.java | 56 ++++++ .../RetentionPolicyCollection.java | 9 + .../ViewRetentionPolicyCollection.java | 56 ++++++ .../definition/RetentionPolicy.java | 4 +- .../execution/RetentionService.java | 92 ---------- .../execution/SchedulingConfig.java | 10 -- .../spi/RetentionPolicyEmptinessChecker.java | 5 - .../src/main/java/module-info.java | 6 +- .../batch/EventSourceRetentionTaskTest.java | 108 ++++++++++++ .../batch/ViewRetentionTaskTest.java | 88 ++++++++++ .../RetentionExecutionDeciderTest.java | 41 +++++ .../RetentionPolicyPartitionerTest.java | 110 ++++++++++++ ...ventSourceRetentionPolicyProviderTest.java | 56 ++++++ .../ViewRetentionPolicyProviderTest.java | 59 +++++++ .../DeletionPolicyCollectionImplTest.java | 59 ------- ...ntSourceRetentionPolicyCollectionTest.java | 84 +++++++++ .../RetentionPolicyCollectionImplTest.java | 72 -------- .../ViewRetentionPolicyCollectionTest.java | 84 +++++++++ .../execution/RetentionServiceTest.java | 101 ----------- .../repositories/PageRepository.java | 3 +- .../entities/CompactionCandidate.java | 12 +- pom.xml | 5 + 82 files changed, 2126 insertions(+), 1015 deletions(-) create mode 100644 ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/CompactionCandidateMapper.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactedPages.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionCandidates.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionPageCapacity.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionStepDefinitions.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTask.java rename ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/{application/services/PaginationCompactionService.java => batch/CompactionWriter.java} (59%) create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTask.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitioner.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorterTest.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java delete mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTaskTest.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriterTest.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTaskTest.java create mode 100644 ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitionerTest.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceFlows.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceService.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/exceptions/MaintenanceJobException.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/services/RetentionPolicyEmptinessChecker.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/valueobjects/DecidedFlowExecutionStatus.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/module-info.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceJobDefinitionTest.java create mode 100644 ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceServiceTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDecider.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciders.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPartitionerConfig.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitioner.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/EventSourceRetentionTask.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/RetentionTask.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/ViewRetentionTask.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/stepdefinition/RetentionStepDefinitions.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProvider.java rename ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/{repositories/DeletionPolicyCollection.java => entities/RetentionPolicyProvider.java} (51%) create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProvider.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/EventSourceRetentionPolicyCollection.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/RetentionPolicyCollection.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/ViewRetentionPolicyCollection.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/EventSourceRetentionTaskTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/ViewRetentionTaskTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciderTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitionerTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProviderTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProviderTest.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventSourceRetentionPolicyCollectionTest.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java create mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewRetentionPolicyCollectionTest.java delete mode 100644 ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java diff --git a/docs/how-to-run.md b/docs/how-to-run.md index 8bb59ef91..d9f4a75fb 100644 --- a/docs/how-to-run.md +++ b/docs/how-to-run.md @@ -126,28 +126,16 @@ Here is an explanation provided for all the possibilities on how to tweak and co */30 * * * * * Fragment Compaction - - ldes-server.compaction-cron - Defines how often the Compaction Service will check the fragments3 - No - 0 0 0 * * * - ldes-server.compaction-duration Defines how long the redundant compacted fragments will remain on the server No PD7 + Maintenance - ldes-server.deletion-cron - Defines how often the redundant compacted fragments will be checked for deletion3 - No - 0 0 0 * * * - - Retention (Retention Policies) - - ldes-server.retention-cron - Defines how often the Retention Service will check the members3 + ldes-server.maintenance-cron + Defines how often the maintenance job will run, which includes retention, compaction and deletion3 No 0 0 0 * * * diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java index d1bf5c5fc..ba0762fb6 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java @@ -43,7 +43,7 @@ public FragmentationService(@Qualifier(ASYNC_JOB_LAUNCHER) JobLauncher jobLaunch this.jobExplorer = jobExplorer; this.jobRepository = jobRepository; this.memberRepository = memberRepository; - this.bucketiseJob = createJob(jobRepository, bucketiseMembersStep, paginationStep); + this.bucketiseJob = createJob(bucketiseMembersStep, paginationStep); this.cleanupOldJobs(); } @@ -77,7 +77,7 @@ private boolean noJobsRunning(ViewName viewName) { }); } - private Job createJob(JobRepository jobRepository, Step step, Step paginationStep) { + private Job createJob(Step step, Step paginationStep) { return new JobBuilder(FRAGMENTATION_JOB, jobRepository) .start(step) .next(paginationStep) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BatchConfiguration.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BatchConfiguration.java index d05bccf3b..cb13f332a 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BatchConfiguration.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/batch/BatchConfiguration.java @@ -10,6 +10,7 @@ @Configuration public class BatchConfiguration { public static final String ASYNC_JOB_LAUNCHER = "asyncJobLauncher"; + @Bean(name = ASYNC_JOB_LAUNCHER) public JobLauncher simpleJobLauncher(JobRepository jobRepository) throws Exception { TaskExecutorJobLauncher jobLauncher = new TaskExecutorJobLauncher(); diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java index f4ca8fdea..679079cbf 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfig.java @@ -12,9 +12,7 @@ public class ServerConfig { private static final String DEFAULT_USE_RELATIVE_URL = "false"; private static final String DEFAULT_MAX_JSONLD_CACHE_CAPACITY = "100"; public static final String HOST_NAME_KEY = "${ldes-server.host-name}"; - public static final String RETENTION_CRON_KEY = "${ldes-server.retention-cron: " + DEFAULT_BACKGROUND_CRON + "}"; - public static final String DELETION_CRON_KEY = "${ldes-server.deletion-cron:" + DEFAULT_BACKGROUND_CRON + "}"; - public static final String COMPACTION_CRON_KEY = "${ldes-server.compaction-cron:" + DEFAULT_BACKGROUND_CRON + "}"; + public static final String MAINTENANCE_CRON_KEY = "${ldes-server.maintenance-cron: " + DEFAULT_BACKGROUND_CRON + "}"; public static final String FRAGMENTATION_CRON = "${ldes-server.fragmentation-cron:" + DEFAULT_FRAGMENTATION_CRON + "}"; public static final String USE_RELATIVE_URL_KEY = "${ldes-server.use-relative-url:" + DEFAULT_USE_RELATIVE_URL + "}"; public static final String MAX_JSONLD_CACHE_CAPACITY = "${ldes-server.max-jsonld-cache-capacity:" + DEFAULT_MAX_JSONLD_CACHE_CAPACITY + "}"; @@ -22,9 +20,8 @@ public class ServerConfig { private String hostName; private String compactionDuration; - private String retentionCron; - private String deletionCron; - private String compactionCron; + private String maintenanceCron; + private String fragmentationCron; private Boolean useRelativeUrl; private Integer maxJsonldCacheCapacity; @@ -36,16 +33,12 @@ public String getCompactionDuration() { return compactionDuration != null ? compactionDuration : DEFAULT_COMPACTION_DURATION; } - public String getRetentionCron() { - return retentionCron != null ? retentionCron : DEFAULT_BACKGROUND_CRON; + public String getMaintenanceCron() { + return maintenanceCron != null ? maintenanceCron : DEFAULT_BACKGROUND_CRON; } - public String getDeletionCron() { - return deletionCron != null ? deletionCron : DEFAULT_BACKGROUND_CRON; - } - - public String getCompactionCron() { - return compactionCron != null ? compactionCron : DEFAULT_BACKGROUND_CRON; + public String getFragmentationCron() { + return fragmentationCron != null ? fragmentationCron : DEFAULT_FRAGMENTATION_CRON; } public Boolean getUseRelativeUrl() { @@ -64,16 +57,12 @@ public void setCompactionDuration(String compactionDuration) { this.compactionDuration = compactionDuration; } - public void setRetentionCron(String retentionCron) { - this.retentionCron = retentionCron; - } - - public void setDeletionCron(String deletionCron) { - this.deletionCron = deletionCron; + public void setMaintenanceCron(String maintenanceCron) { + this.maintenanceCron = maintenanceCron; } - public void setCompactionCron(String compactionCron) { - this.compactionCron = compactionCron; + public void setFragmentationCron(String fragmentationCron) { + this.fragmentationCron = fragmentationCron; } public void setUseRelativeUrl(Boolean useRelativeUrl) { diff --git a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfigTest.java b/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfigTest.java index 84cfc35cb..cd70e9fb4 100644 --- a/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfigTest.java +++ b/ldes-server-domain/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/constants/ServerConfigTest.java @@ -11,29 +11,27 @@ void when_PropertiesAreFilledIn_TheyCanBeConsulted() { serverConfig.setHostName("LOCALHOST"); serverConfig.setUseRelativeUrl(true); serverConfig.setCompactionDuration("PT1M"); - serverConfig.setRetentionCron("*/20 * * * * *"); - serverConfig.setDeletionCron("*/30 * * * * *"); - serverConfig.setCompactionCron("*/45 * * * * *"); + serverConfig.setFragmentationCron("*/20 * * * * *"); + serverConfig.setMaintenanceCron("*/30 * * * * *"); assertThat(serverConfig.getHostName()).isEqualTo("LOCALHOST"); assertThat(serverConfig.getUseRelativeUrl()).isTrue(); assertThat(serverConfig.getCompactionDuration()).isEqualTo("PT1M"); - assertThat(serverConfig.getRetentionCron()).isEqualTo("*/20 * * * * *"); - assertThat(serverConfig.getDeletionCron()).isEqualTo("*/30 * * * * *"); - assertThat(serverConfig.getCompactionCron()).isEqualTo("*/45 * * * * *"); + assertThat(serverConfig.getFragmentationCron()).isEqualTo("*/20 * * * * *"); + assertThat(serverConfig.getMaintenanceCron()).isEqualTo("*/30 * * * * *"); } @Test void when_PropertiesAreEmpty_NullOrDefaultValuesAreReturned() { final String backgroundCron = "0 0 0 * * *"; + final String fragmentationCron = "*/30 * * * * *"; ServerConfig serverConfig = new ServerConfig(); assertThat(serverConfig.getHostName()).isNull(); assertThat(serverConfig.getUseRelativeUrl()).isFalse(); assertThat(serverConfig.getCompactionDuration()).isEqualTo("P7D"); - assertThat(serverConfig.getRetentionCron()).isEqualTo(backgroundCron); - assertThat(serverConfig.getDeletionCron()).isEqualTo(backgroundCron); - assertThat(serverConfig.getCompactionCron()).isEqualTo(backgroundCron); + assertThat(serverConfig.getMaintenanceCron()).isEqualTo(backgroundCron); + assertThat(serverConfig.getFragmentationCron()).isEqualTo(fragmentationCron); } } \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java index 56a2cd20b..ee871d9c3 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PagePostgresRepository.java @@ -5,6 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.entities.Page; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.batch.PaginationRowMapper; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper.CompactionCandidateMapper; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; @@ -14,7 +15,6 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.stream.Stream; @Repository public class PagePostgresRepository implements PageRepository { @@ -68,10 +68,12 @@ public void markAllPagesImmutableByCollectionName(String collectionName) { } @Override - public Stream getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage) { - return pageEntityRepository.findCompactionCandidates(viewName.getCollectionName(), viewName.getViewName(), capacityPerPage) - .stream().map(projection -> new CompactionCandidate(projection.getFragmentId(), projection.getSize(), projection.getToPage(), - projection.getImmutable(), projection.getExpiration(), projection.getBucketId(), projection.getPartialUrl())); + public List getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage) { + return pageEntityRepository + .findCompactionCandidates(viewName.getCollectionName(), viewName.getViewName(), capacityPerPage) + .stream() + .map(CompactionCandidateMapper::fromProjection) + .toList(); } @Override diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/CompactionCandidateMapper.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/CompactionCandidateMapper.java new file mode 100644 index 000000000..adf52693c --- /dev/null +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/mapper/CompactionCandidateMapper.java @@ -0,0 +1,17 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.mapper; + +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection.CompactionCandidateProjection; + +public class CompactionCandidateMapper { + private CompactionCandidateMapper() {} + + public static CompactionCandidate fromProjection(CompactionCandidateProjection projection) { + return new CompactionCandidate( + projection.getFragmentId(), + projection.getSize(), + projection.getToPage(), + projection.getBucketId(), + projection.getPartialUrl()); + } +} diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/projection/CompactionCandidateProjection.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/projection/CompactionCandidateProjection.java index 0b73227f2..38f3fbc68 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/projection/CompactionCandidateProjection.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/projection/CompactionCandidateProjection.java @@ -1,13 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.projection; -import java.time.LocalDateTime; - public interface CompactionCandidateProjection { Long getFragmentId(); Integer getSize(); Long getToPage(); - Boolean getImmutable(); - LocalDateTime getExpiration(); Long getBucketId(); String getPartialUrl(); } diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 289e4a77f..f6ae40064 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -32,13 +32,13 @@ JOIN views USING (view_id) """, nativeQuery = true) void markAllPagesImmutableByCollectionName(String collectionName); - @Query(value = "SELECT p.id as fragmentId, COUNT(*) AS size, r.toPage.id AS toPage, p.immutable AS immutable, " + - "p.expiration AS expiration, " + - "p.bucket.bucketId AS bucketId, p.partialUrl AS partialUrl " + - "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN PageRelationEntity r ON p = r.fromPage " + - "WHERE v.eventStream.name = :collectionName AND v.name = :viewName " + - "GROUP BY p.id, r.toPage.id " + - "HAVING COUNT(*) < :capacityPerPage") + @Query(value = "SELECT p.id as fragmentId, COUNT(*) AS size, r.toPage.id AS toPage, p.bucket.bucketId AS bucketId, p.partialUrl AS partialUrl " + + "FROM PageEntity p JOIN BucketEntity b ON p.bucket = b JOIN ViewEntity v ON b.view = v JOIN PageRelationEntity r ON p = r.fromPage " + + "WHERE v.eventStream.name = :collectionName " + + "AND v.name = :viewName " + + "AND p.expiration IS NULL AND p.immutable " + + "GROUP BY p.id, r.toPage.id " + + "HAVING COUNT(*) < :capacityPerPage") @Transactional(readOnly = true) List findCompactionCandidates(@Param("collectionName") String collectionName, @Param("viewName") String viewName, diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java index dc53e37f5..7bc57fbf0 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java @@ -30,13 +30,15 @@ @SpringBootTest @AutoConfigureMockMvc @AutoConfigureEmbeddedDatabase(type = AutoConfigureEmbeddedDatabase.DatabaseType.POSTGRES, - refresh = AutoConfigureEmbeddedDatabase.RefreshMode.AFTER_EACH_TEST_METHOD, replace = AutoConfigureEmbeddedDatabase.Replace.ANY) @ActiveProfiles("postgres-test") -@ContextConfiguration(classes = { MemberEntityRepository.class, PageRelationPostgresRepository.class, PageRelationEntityRepository.class }) -@ComponentScan(value = { "be.vlaanderen.informatievlaanderen.ldes.server" }) -@TestPropertySource(properties = { "ldes-server.fragmentation-cron=*/1 * * * * *", "ldes-server.compaction-cron=*/10 * * * * *", - "ldes-server.deletion-cron=*/20 * * * * *", "ldes-server.compaction-duration=PT1S" }) +@ContextConfiguration(classes = {MemberEntityRepository.class, PageRelationPostgresRepository.class, PageRelationEntityRepository.class}) +@ComponentScan(value = {"be.vlaanderen.informatievlaanderen.ldes.server"}) +@TestPropertySource(properties = { + "ldes-server.fragmentation-cron=*/1 * * * * *", + "ldes-server.maintenance-cron=*/10 * * * * *", + "ldes-server.compaction-duration=PT1S" +}) @SuppressWarnings("java:S2187") public class LdesServerIntegrationTest { @Autowired diff --git a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml index b19995f7f..6e51b49ad 100644 --- a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml +++ b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml @@ -1,9 +1,8 @@ ldes-server: host-name: "http://localhost:8080" compaction-duration: "*/10 * * * * *" - retention-cron: "*/10 * * * * *" - deletion-cron: "*/5 * * * * *" fragmentation-cron: "*/10 * * * * *" + maintenance-cron: "*/10 * * * * *" springdoc.swaggerui.path: "/swagger" management: diff --git a/ldes-server-integration-test/src/test/resources/features/retention/retention.feature b/ldes-server-integration-test/src/test/resources/features/retention/retention.feature index 763bac5b3..3444ac21e 100644 --- a/ldes-server-integration-test/src/test/resources/features/retention/retention.feature +++ b/ldes-server-integration-test/src/test/resources/features/retention/retention.feature @@ -39,9 +39,9 @@ Feature: LDES Server Retention Then the LDES contains members Examples: - | eventStreamDescriptionFile | template | collection | expectedMemberCount | - | "data/input/eventstreams/deletion/mobility-hindrances_versionbased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 10 | - | "data/input/eventstreams/deletion/mobility-hindrances_versionbased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 30 | + | eventStreamDescriptionFile | template | collection | expectedMemberCount | + | "data/input/eventstreams/deletion/mobility-hindrances_versionbased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 10 | + | "data/input/eventstreams/deletion/mobility-hindrances_versionbased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 30 | @deletion @time-based Scenario Outline: Server retention time-based to event source @@ -50,8 +50,8 @@ Feature: LDES Server Retention Then the LDES contains members Examples: - | eventStreamDescriptionFile | template | collection | expectedMemberCount | - | "data/input/eventstreams/deletion/mobility-hindrances_timebased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 0 | + | eventStreamDescriptionFile | template | collection | expectedMemberCount | + | "data/input/eventstreams/deletion/mobility-hindrances_timebased_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 0 | @deletion @version-based-and-time-based Scenario Outline: Server retention combined-based to event source @@ -60,5 +60,5 @@ Feature: LDES Server Retention Then the LDES contains members Examples: - | eventStreamDescriptionFile | template | collection | expectedMemberCount | - | "data/input/eventstreams/deletion/mobility-hindrances_combined_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 5 | + | eventStreamDescriptionFile | template | collection | expectedMemberCount | + | "data/input/eventstreams/deletion/mobility-hindrances_combined_with_eventsource.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 5 | diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java deleted file mode 100644 index 21ca0dc4d..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateService.java +++ /dev/null @@ -1,49 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.ViewCapacity; -import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import org.springframework.stereotype.Component; - -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import static be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.CompactionCandidateSorter.getCompactionCandidateList; - -@Component -public class CompactionCandidateService { - private final PageRepository pageRepository; - - - public CompactionCandidateService(PageRepository pageRepository) { - this.pageRepository = pageRepository; - } - - /** - * Preparation step for the Compaction Service. - * Will retrieve an aggregated view of the allocation table where the size of members is lower than the - * maximum view capacity. - * If these fragments are connected, these will be returned in the same group. - * - * @param viewCapacity Contains the name and capacity for a view - * @return a structured group of Fragments for a view that can be compacted - */ - public Collection> getCompactionTaskList(ViewCapacity viewCapacity) { - List possibleCompactionCandidates = getPossibleCompactionCandidates(viewCapacity); - - if (possibleCompactionCandidates.isEmpty()) { - return List.of(); - } - - return getCompactionCandidateList(possibleCompactionCandidates, viewCapacity.getCapacityPerPage()); - } - - protected List getPossibleCompactionCandidates(ViewCapacity viewCapacity) { - var compactionCandidates = pageRepository.getPossibleCompactionCandidates(viewCapacity.getViewName(), viewCapacity.getCapacityPerPage()) - .toList(); - return compactionCandidates.stream() - .filter(CompactionCandidate::isCompactable) - .toList(); - } -} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java index 2f4eff7a6..d008ebfac 100644 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorter.java @@ -1,56 +1,52 @@ package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects.CompactedPages; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects.CompactionCandidates; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects.CompactionPageCapacity; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import org.springframework.stereotype.Component; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; +@Component public class CompactionCandidateSorter { - private CompactionCandidateSorter() { - } - - public static Collection> getCompactionCandidateList(List candidatesList, int capacity) { - - List firstElements = candidatesList.stream() - .filter(candidate -> hasNoConnections(candidate, candidatesList.stream() - .toList())).toList(); - - AtomicInteger index = new AtomicInteger(); - Map> compactionDesign = new HashMap<>(); - - firstElements.forEach(candidate -> { - int currentCapacity = 0; - Set splitList = new HashSet<>(); - Optional currentCandidate = Optional.of(candidate); - while (currentCandidate.isPresent()) { - currentCapacity += currentCandidate.get().getSize(); - if (currentCapacity > capacity) { - currentCapacity = 0; - if(splitList.size() > 1) { - compactionDesign.put(index.incrementAndGet(), splitList); - splitList = new HashSet<>(); - } - } - else { - splitList.add(currentCandidate.get()); - } - long nextId = currentCandidate.get().getNextPageId(); - currentCandidate = candidatesList.stream().filter(c->c.getId() == nextId).findFirst(); - } - - if(splitList.size() > 1) { - compactionDesign.put(index.incrementAndGet(), splitList); - } - }); - - return compactionDesign.values(); - } - - public static boolean hasNoConnections(CompactionCandidate fragment, List fragments) { - return fragments.stream().noneMatch(fragment1 -> isConnectedTo(fragment1, fragment)); - } - - public static boolean isConnectedTo(CompactionCandidate treeNode, CompactionCandidate otherTreeNode) { - return treeNode.getNextPageId() == otherTreeNode.getId(); - } + + public List> getSortedCompactionCandidates(List candidates, + int capacityPerPage) { + final CompactionCandidates compactionCandidates = new CompactionCandidates(candidates); + final CompactionPageCapacity compactionPageCapacity = new CompactionPageCapacity(capacityPerPage); + return compactionCandidates + .getLeadingPages().stream() + .flatMap(leadingPage -> getCompactedCandidatesForLeadingPage(leadingPage, compactionCandidates, compactionPageCapacity).stream()) + .toList(); + } + + private List> getCompactedCandidatesForLeadingPage(CompactionCandidate leadingPage, + CompactionCandidates candidates, + CompactionPageCapacity capacity) { + CompactedPages compactedPages = new CompactedPages(); + Optional currentCandidate = Optional.of(leadingPage); + + while (currentCandidate.isPresent()) { + addCandidatesToCapacity(capacity, currentCandidate.get(), compactedPages); + currentCandidate = candidates.getNextCandidate(currentCandidate.get().getNextPageId()); + } + + compactedPages.addCandidatesToCompactedPages(); + + return compactedPages.getPages(); + } + + private void addCandidatesToCapacity(CompactionPageCapacity compactionPageCapacity, + CompactionCandidate candidate, + CompactedPages compactedPages) { + compactionPageCapacity.increase(candidate.getSize()); + if (compactionPageCapacity.exceedsMaxCapacity()) { + compactionPageCapacity.reset(); + compactedPages.addCandidatesToCompactedPages(); + } else { + compactedPages.addCompactionCandidate(candidate); + } + } + } diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java deleted file mode 100644 index b2dad97f6..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionScheduler.java +++ /dev/null @@ -1,52 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.repository.ViewCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.spi.RetentionPolicyEmptinessChecker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConfig.COMPACTION_CRON_KEY; - -@Service -public class CompactionScheduler { - private static final Logger LOGGER = LoggerFactory.getLogger(CompactionScheduler.class); - private final ViewCollection viewCollection; - private final PaginationCompactionService paginationCompactionService; - private final CompactionCandidateService compactionCandidateService; - private final RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker; - - public CompactionScheduler(ViewCollection viewCollection, - PaginationCompactionService paginationCompactionService, - CompactionCandidateService compactionCandidateService, - RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker) { - this.viewCollection = viewCollection; - this.paginationCompactionService = paginationCompactionService; - this.compactionCandidateService = compactionCandidateService; - this.retentionPolicyEmptinessChecker = retentionPolicyEmptinessChecker; - } - - @SuppressWarnings("java:S6857") - @Scheduled(cron = COMPACTION_CRON_KEY) - public void compactFragments() { - if (retentionPolicyEmptinessChecker.isEmpty()) { - LOGGER.info("Compaction skipped: no retention policies found."); - return; - } - viewCollection.getAllViewCapacities() - .parallelStream() - .forEach(viewCapacity -> { - - var compactionTaskList = compactionCandidateService.getCompactionTaskList(viewCapacity); - - if (compactionTaskList.isEmpty()) { - LOGGER.info("No compaction candidates available for {}", viewCapacity.getViewName().getViewName()); - } else { - LOGGER.info("Processing {} compaction candidates available for {}", compactionTaskList.size(), viewCapacity.getViewName().getViewName()); - - compactionTaskList.forEach(paginationCompactionService::applyCompactionForFragments); - } - }); - } -} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java deleted file mode 100644 index cc84ed01b..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionScheduler.java +++ /dev/null @@ -1,34 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.spi.RetentionPolicyEmptinessChecker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConfig.DELETION_CRON_KEY; - -@Service -public class FragmentDeletionScheduler { - private static final Logger LOGGER = LoggerFactory.getLogger(FragmentDeletionScheduler.class); - private final PageRepository pageRepository; - private final RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker; - - public FragmentDeletionScheduler(PageRepository pageRepository, RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker) { - this.pageRepository = pageRepository; - this.retentionPolicyEmptinessChecker = retentionPolicyEmptinessChecker; - } - - @SuppressWarnings("java:S6857") - @Scheduled(cron = DELETION_CRON_KEY) - public void deleteFragments() { - if(retentionPolicyEmptinessChecker.isEmpty()) { - LOGGER.atDebug().log("Fragment deletion skipped: no retention policies found."); - return; - } - pageRepository.deleteOutdatedFragments(LocalDateTime.now()); - } -} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java deleted file mode 100644 index 0b79fad27..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentationConfigCompaction.java +++ /dev/null @@ -1,15 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategy; -import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationStrategyImpl; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FragmentationConfigCompaction { - - @Bean("compactionFragmentation") - public FragmentationStrategy compactionFragmentationStrategy() { - return new FragmentationStrategyImpl(); - } -} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java deleted file mode 100644 index 06292ce3e..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/SchedulingConfigCompaction.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; - -@Configuration -@EnableScheduling -public class SchedulingConfigCompaction { - -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactedPages.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactedPages.java new file mode 100644 index 000000000..955001e7e --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactedPages.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class CompactedPages { + private final List> pages; + private final Set compactedCandidatesToAdd; + + + public CompactedPages() { + pages = new ArrayList<>(); + compactedCandidatesToAdd = new HashSet<>(); + } + + public void addCandidatesToCompactedPages() { + if (!compactedCandidatesToAdd.isEmpty()) { + pages.add(Set.copyOf(compactedCandidatesToAdd)); + compactedCandidatesToAdd.clear(); + } + } + + public void addCompactionCandidate(CompactionCandidate candidate) { + compactedCandidatesToAdd.add(candidate); + } + + public List> getPages() { + return List.copyOf(pages); + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionCandidates.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionCandidates.java new file mode 100644 index 000000000..0213d8ce4 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionCandidates.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; + +import java.util.List; +import java.util.Optional; + +public class CompactionCandidates { + private final List candidates; + + public CompactionCandidates(List candidates) { + this.candidates = candidates; + } + + public List getLeadingPages() { + return candidates.stream() + .filter(candidate -> hasNoConnections(candidate, List.copyOf(candidates))) + .toList(); + } + + public Optional getNextCandidate(long nextId) { + return candidates.stream() + .filter(candidate -> candidate.getId() == nextId) + .findFirst(); + } + + private static boolean hasNoConnections(CompactionCandidate candidate, List pages) { + return pages.stream().noneMatch(page -> isConnectedTo(page, candidate)); + } + + private static boolean isConnectedTo(CompactionCandidate treeNode, CompactionCandidate otherTreeNode) { + return treeNode.getNextPageId() == otherTreeNode.getId(); + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionPageCapacity.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionPageCapacity.java new file mode 100644 index 000000000..8ad0ad834 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/valueobjects/CompactionPageCapacity.java @@ -0,0 +1,22 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.valueobjects; + +public class CompactionPageCapacity { + private final int maxCapacity; + private int currentCapacity; + + public CompactionPageCapacity(int maxCapacity) { + this.maxCapacity = maxCapacity; + } + + public void reset() { + currentCapacity = 0; + } + + public boolean exceedsMaxCapacity() { + return currentCapacity > maxCapacity; + } + + public void increase(int delta) { + currentCapacity += delta; + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionStepDefinitions.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionStepDefinitions.java new file mode 100644 index 000000000..4531b4474 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionStepDefinitions.java @@ -0,0 +1,42 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.partition.support.Partitioner; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +public class CompactionStepDefinitions { + public static final String COMPACTION_STEP = "compactionStep"; + public static final String SINGLE_VIEW_COMPACTION_STEP = "singleViewCompactionStep"; + public static final String SINGLE_VIEW_COMPACTION_PARTITIONER = "singleViewCompactionPartitioner"; + public static final String DELETION_STEP = "deletionStep"; + + @Bean + public Step compactionStep(JobRepository jobRepository, + Partitioner viewCapacityPartitioner, + Tasklet compactionTask, + PlatformTransactionManager transactionManager) { + return new StepBuilder(COMPACTION_STEP, jobRepository) + .partitioner(SINGLE_VIEW_COMPACTION_PARTITIONER, viewCapacityPartitioner) + .step(new StepBuilder(SINGLE_VIEW_COMPACTION_STEP, jobRepository) + .tasklet(compactionTask, transactionManager) + .build()) + .taskExecutor(new SimpleAsyncTaskExecutor()) + .build(); + } + + @Bean + public Step deletionStep(JobRepository jobRepository, + Tasklet pageDeletionTask, + PlatformTransactionManager transactionManager) { + return new StepBuilder(DELETION_STEP, jobRepository) + .tasklet(pageDeletionTask, transactionManager) + .build(); + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTask.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTask.java new file mode 100644 index 000000000..2368c7944 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTask.java @@ -0,0 +1,43 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.CompactionCandidateSorter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class CompactionTask implements Tasklet { + private final PageRepository pageRepository; + private final be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.CompactionCandidateSorter compactionCandidateSorter; + private final CompactionWriter compactionWriter; + + public CompactionTask(PageRepository pageRepository, CompactionCandidateSorter compactionCandidateSorter, CompactionWriter compactionWriter) { + this.pageRepository = pageRepository; + this.compactionCandidateSorter = compactionCandidateSorter; + this.compactionWriter = compactionWriter; + } + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + final ExecutionContext executionContext = chunkContext.getStepContext().getStepExecution().getExecutionContext(); + final ViewName viewName = ViewName.fromString(executionContext.getString("viewName")); + final int capacityPerPage = executionContext.getInt("capacityPerPage"); + + final List compactionCandidates = pageRepository.getPossibleCompactionCandidates(viewName, capacityPerPage); + + + compactionCandidateSorter + .getSortedCompactionCandidates(compactionCandidates, capacityPerPage) + .forEach(compactionWriter::write); + + return RepeatStatus.FINISHED; + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriter.java similarity index 59% rename from ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java rename to ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriter.java index 632b90251..0d21bf3a5 100644 --- a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionService.java +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriter.java @@ -1,5 +1,6 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.PageDeletionTimeSetter; import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.CompactedFragmentCreator; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; @@ -12,27 +13,30 @@ import java.util.Set; @Component -public class PaginationCompactionService { +public class CompactionWriter { private final PageRelationRepository pageRelationRepository; private final PageMemberRepository pageMemberRepository; private final CompactedFragmentCreator compactedFragmentCreator; private final PageDeletionTimeSetter pageDeletionTimeSetter; private final ObservationRegistry observationRegistry; - public PaginationCompactionService(PageRelationRepository pageRelationRepository, PageMemberRepository pageMemberRepository, - CompactedFragmentCreator compactedFragmentCreator, PageDeletionTimeSetter pageDeletionTimeSetter, ObservationRegistry observationRegistry) { - this.pageRelationRepository = pageRelationRepository; - this.pageMemberRepository = pageMemberRepository; - this.compactedFragmentCreator = compactedFragmentCreator; - this.pageDeletionTimeSetter = pageDeletionTimeSetter; - this.observationRegistry = observationRegistry; + public CompactionWriter(PageRelationRepository pageRelationRepository, + PageMemberRepository pageMemberRepository, + CompactedFragmentCreator compactedFragmentCreator, + PageDeletionTimeSetter pageDeletionTimeSetter, + ObservationRegistry observationRegistry) { + this.pageRelationRepository = pageRelationRepository; + this.pageMemberRepository = pageMemberRepository; + this.compactedFragmentCreator = compactedFragmentCreator; + this.pageDeletionTimeSetter = pageDeletionTimeSetter; + this.observationRegistry = observationRegistry; } - public void applyCompactionForFragments(Set toBeCompactedFragments) { + public void write(Set toBeCompactedPages) { Observation compactionObservation = Observation.createNotStarted("compaction", observationRegistry).start(); - long compactedFragmentId = compactedFragmentCreator.createCompactedPage(toBeCompactedFragments); - List compactedPageIds = toBeCompactedFragments.stream().map(CompactionCandidate::getId).toList(); + long compactedFragmentId = compactedFragmentCreator.createCompactedPage(toBeCompactedPages); + List compactedPageIds = toBeCompactedPages.stream().map(CompactionCandidate::getId).toList(); pageMemberRepository.setPageMembersToNewPage(compactedFragmentId, compactedPageIds); pageRelationRepository.updateCompactionBucketRelations(compactedPageIds, compactedFragmentId); diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTask.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTask.java new file mode 100644 index 000000000..e3ebf7171 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTask.java @@ -0,0 +1,24 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; + +@Component +public class PageDeletionTask implements Tasklet { + private final PageRepository pageRepository; + + public PageDeletionTask(PageRepository pageRepository) { + this.pageRepository = pageRepository; + } + + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + pageRepository.deleteOutdatedFragments(LocalDateTime.now()); + return RepeatStatus.FINISHED; + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitioner.java b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitioner.java new file mode 100644 index 000000000..a7b6fa160 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitioner.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.repository.ViewCollection; +import org.springframework.batch.core.partition.support.Partitioner; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.stream.Collectors; + +@Component +public class ViewCapacityPartitioner implements Partitioner { + private final ViewCollection viewCollection; + + public ViewCapacityPartitioner(ViewCollection viewCollection) { + this.viewCollection = viewCollection; + } + + @Override + public Map partition(int gridSize) { + return viewCollection.getAllViewCapacities().stream() + .collect(Collectors.toMap( + viewCapacity -> "view:%s".formatted(viewCapacity.getViewName().asString()), + viewCapacity -> new ExecutionContext(Map.of( + "viewName", viewCapacity.getViewName().asString(), + "capacityPerPage", viewCapacity.getCapacityPerPage() + )) + )); + } +} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java deleted file mode 100644 index f45c5ee74..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateServiceTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.ViewCapacity; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.time.LocalDateTime; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class CompactionCandidateServiceTest { - PageRepository pagePostgresRepository = mock(PageRepository.class); - CompactionCandidateService service = new CompactionCandidateService(pagePostgresRepository); - - List candidates = List.of( - new CompactionCandidate(1L, 2, 2L, true, null, 1L, "/ex/p"), - new CompactionCandidate(2L, 3, 3L, true, null, 1L, "/ex/p"), - new CompactionCandidate(3L, 8, 4L, true, null, 1L, "/ex/p"), - new CompactionCandidate(4L, 3, 5L, true, null, 1L, "/ex/p"), - new CompactionCandidate(5L, 3, 6L, true, null, 1L, "/ex/p"), - new CompactionCandidate(6L, 2, 7L, true, LocalDateTime.now(), 1L, "/ex/p"), - new CompactionCandidate(7L, 2, 8L, false, null, 1L, "/ex/p") - ); - - @BeforeEach - void setup() { - when(pagePostgresRepository.getPossibleCompactionCandidates(any(), anyInt())) - .thenReturn(candidates.stream()); - } - - @Test - void testGetPossibleCompactionCandidates() { - var compactionCandidates = service.getPossibleCompactionCandidates( - new ViewCapacity(ViewName.fromString("ex/p"), 10)); - - assertEquals(5, compactionCandidates.size()); - assertEquals(1L, compactionCandidates.get(0).getId()); - assertEquals(2L, compactionCandidates.get(1).getId()); - assertEquals(3L, compactionCandidates.get(2).getId()); - assertEquals(4L, compactionCandidates.get(3).getId()); - assertEquals(5L, compactionCandidates.get(4).getId()); - } - - @Test - void testGetCompactionTaskList() { - var taskList = service.getCompactionTaskList(new ViewCapacity(ViewName.fromString("c/v"), 10)); - - assertEquals(2, taskList.size()); - - var firstCompaction = taskList.stream().toList().get(0); - assertEquals(2, firstCompaction.size()); - assertTrue(firstCompaction.stream().anyMatch(candidate -> candidate.getId() == 1L)); - assertTrue(firstCompaction.stream().anyMatch(candidate -> candidate.getId() == 2L)); - - var secondCompaction = taskList.stream().toList().get(1); - assertEquals(2, secondCompaction.size()); - assertTrue(secondCompaction.stream().anyMatch(candidate -> candidate.getId() == 4L)); - assertTrue(secondCompaction.stream().anyMatch(candidate -> candidate.getId() == 5L)); - } -} diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorterTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorterTest.java new file mode 100644 index 000000000..3373ea1ca --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionCandidateSorterTest.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; + +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class CompactionCandidateSorterTest { + + private static final List compactableCandidates = List.of( + new CompactionCandidate(1L, 2, 2L, 1L, "/ex/p"), + new CompactionCandidate(2L, 3, 3L, 1L, "/ex/p"), + new CompactionCandidate(3L, 8, 4L, 1L, "/ex/p"), + new CompactionCandidate(4L, 3, 5L, 1L, "/ex/p"), + new CompactionCandidate(5L, 3, 6L, 1L, "/ex/p") + ); + + + @Test + void testGetCompactionTaskList() { + final Collection> taskList = new CompactionCandidateSorter() + .getSortedCompactionCandidates(compactableCandidates, 10); + + assertThat(taskList) + .hasSize(2) + .allSatisfy(set -> assertThat(set).hasSize(2)) + .map(set -> set.stream().map(CompactionCandidate::getId).sorted().toList()) + .containsExactlyInAnyOrder(List.of(1L, 2L), List.of(4L, 5L)); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java deleted file mode 100644 index 567a98128..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/CompactionSchedulerTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.repository.ViewCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.RetentionPolicyCollection; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -class CompactionSchedulerTest { - @Mock - private ViewCollection viewCollection; - @Mock - private PaginationCompactionService paginationCompactionService; - @Mock - private CompactionCandidateService compactionCandidateService; - @Mock - private RetentionPolicyCollection retentionPolicyCollection; - - @InjectMocks - private CompactionScheduler compactionScheduler; - - - @Test - void given_RetentionPoliciesCollectionIsEmpty_when_CompactFragments_then_DoNotRun() { - when(retentionPolicyCollection.isEmpty()).thenReturn(true); - - compactionScheduler.compactFragments(); - - verify(retentionPolicyCollection).isEmpty(); - verifyNoMoreInteractions(viewCollection, paginationCompactionService, compactionCandidateService, retentionPolicyCollection); - } - - @Test - void given_RetentionPoliciesCollectionIsNotEmpty_when_CompactFragments_then_DoRun() { - when(retentionPolicyCollection.isEmpty()).thenReturn(false); - - compactionScheduler.compactFragments(); - - verify(retentionPolicyCollection).isEmpty(); - verify(viewCollection).getAllViewCapacities(); - verifyNoMoreInteractions(viewCollection, paginationCompactionService, compactionCandidateService, retentionPolicyCollection); - } - - -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java deleted file mode 100644 index 3a973f092..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/FragmentDeletionSchedulerTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.spi.RetentionPolicyEmptinessChecker; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -class FragmentDeletionSchedulerTest { - @Mock - private RetentionPolicyEmptinessChecker retentionPolicyEmptinessChecker; - @Mock - private PageRepository pageRepository; - private FragmentDeletionScheduler fragmentDeletionScheduler; - - @BeforeEach - void setUp() { - fragmentDeletionScheduler = new FragmentDeletionScheduler(pageRepository, retentionPolicyEmptinessChecker); - } - - @Test - void when_FragmentHasDeleteTimeEarlierThanCurrentTime_then_ItIsDeletedAndEventIsSent() { - when(retentionPolicyEmptinessChecker.isEmpty()).thenReturn(false); - - fragmentDeletionScheduler.deleteFragments(); - - verify(pageRepository).deleteOutdatedFragments(any()); - } - - @Test - void given_RetentionPolicyCollectionIsEmpty_when_CompactFragments_then_DoNotRun() { - when(retentionPolicyEmptinessChecker.isEmpty()).thenReturn(true); - - fragmentDeletionScheduler.deleteFragments(); - - verifyNoInteractions(pageRepository); - } -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java deleted file mode 100644 index f17585353..000000000 --- a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/application/services/PaginationCompactionServiceTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services; - -import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.CompactedFragmentCreator; -import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; -import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; -import io.micrometer.observation.ObservationRegistry; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.InOrder; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.mockito.Mockito.*; - -class PaginationCompactionServiceTest { - private final PageRelationRepository pageRelationRepository = mock(PageRelationRepository.class); - private final PageMemberRepository pageMemberRepository = mock(PageMemberRepository.class); - private final CompactedFragmentCreator compactedFragmentCreator = mock(CompactedFragmentCreator.class); - private final PageDeletionTimeSetter pageDeletionTimeSetter = mock(PageDeletionTimeSetter.class); - private final ObservationRegistry observationRegistry = ObservationRegistry.create(); - private PaginationCompactionService paginationCompactionService; - private Set candidates; - - @BeforeEach - void setUp() { - candidates = new HashSet<>(); - paginationCompactionService = new PaginationCompactionService(pageRelationRepository, pageMemberRepository, - compactedFragmentCreator, pageDeletionTimeSetter, observationRegistry); - } - - @Test - void when_CompactPages_Then_PagesAreCompacted() { - List ids = List.of(3L, 2L, 1L); - long newId = 10L; - candidates.add(new CompactionCandidate(1L, 5, 2L, true, - null, 1L, "http://example.com")); - candidates.add(new CompactionCandidate(2L, 5, 3L, true, - null, 1L, "http://example.com")); - candidates.add(new CompactionCandidate(3L, 5, 4L, true, - null, 1L, "http://example.com")); - when(compactedFragmentCreator.createCompactedPage(candidates)).thenReturn(newId); - - paginationCompactionService.applyCompactionForFragments(candidates); - - InOrder inOrder = inOrder(pageRelationRepository, pageMemberRepository, - compactedFragmentCreator, pageDeletionTimeSetter); - inOrder.verify(compactedFragmentCreator).createCompactedPage(candidates); - inOrder.verify(pageMemberRepository).setPageMembersToNewPage(newId, ids); - inOrder.verify(pageRelationRepository).updateCompactionBucketRelations(ids, newId); - inOrder.verify(pageDeletionTimeSetter).setDeleteTimeOfFragment(ids); - inOrder.verifyNoMoreInteractions(); - } -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTaskTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTaskTest.java new file mode 100644 index 000000000..0bf617684 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionTaskTest.java @@ -0,0 +1,67 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.CompactionCandidateSorter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.scope.context.StepContext; +import org.springframework.batch.item.ExecutionContext; + +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class CompactionTaskTest { + @Mock + private PageRepository pageRepository; + @Mock + private CompactionCandidateSorter compactionCandidateSorter; + @Mock + private CompactionWriter compactionWriter; + @Mock + private StepExecution stepExecution; + @Mock + private ExecutionContext executionContext; + @InjectMocks + private CompactionTask compactionTask; + + @BeforeEach + void setUp() { + when(stepExecution.getExecutionContext()).thenReturn(executionContext); + } + + @Test + void given_RetentionPoliciesCollectionIsNotEmpty_when_CompactFragments_then_DoRun() { + final ViewName viewName = new ViewName("collection", "view"); + final int capacityPerPage = 125; + final List candidates = IntStream.range(0, 5) + .mapToObj(i -> mock(CompactionCandidate.class)) + .toList(); + final List> taskList = IntStream.range(0, 5) + .boxed() + .map(i -> Set.of()) + .toList(); + + when(executionContext.getString("viewName")).thenReturn(viewName.asString()); + when(executionContext.getInt("capacityPerPage")).thenReturn(capacityPerPage); + when(pageRepository.getPossibleCompactionCandidates(viewName, capacityPerPage)).thenReturn(candidates); + when(compactionCandidateSorter.getSortedCompactionCandidates(candidates, capacityPerPage)).thenReturn(taskList); + + compactionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + + verify(pageRepository).getPossibleCompactionCandidates(viewName, capacityPerPage); + verify(compactionCandidateSorter).getSortedCompactionCandidates(candidates, capacityPerPage); + verify(compactionWriter, times(taskList.size())).write(anySet()); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriterTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriterTest.java new file mode 100644 index 000000000..270a9c367 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/CompactionWriterTest.java @@ -0,0 +1,59 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.application.services.PageDeletionTimeSetter; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.CompactedFragmentCreator; +import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.CompactionCandidate; +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRelationRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; +import io.micrometer.observation.ObservationRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.*; + +class CompactionWriterTest { + private PageRelationRepository pageRelationRepository; + private PageMemberRepository pageMemberRepository; + private CompactedFragmentCreator compactedFragmentCreator; + private PageDeletionTimeSetter pageDeletionTimeSetter; + private CompactionWriter compactionWriter; + + @BeforeEach + void setUp() { + pageRelationRepository = mock(); + pageMemberRepository = mock(); + compactedFragmentCreator = mock(); + pageDeletionTimeSetter = mock(); + compactionWriter = new CompactionWriter(pageRelationRepository, pageMemberRepository, compactedFragmentCreator, pageDeletionTimeSetter, ObservationRegistry.NOOP); + } + + @Test + void when_CompactPages_Then_PagesAreCompacted() { + List ids = List.of(3L, 2L, 1L); + long newId = 10L; + final Set candidates = createCompactionCandidates(); + when(compactedFragmentCreator.createCompactedPage(candidates)).thenReturn(newId); + + compactionWriter.write(candidates); + + InOrder inOrder = inOrder(compactedFragmentCreator, pageMemberRepository, pageRelationRepository, pageDeletionTimeSetter); + inOrder.verify(compactedFragmentCreator).createCompactedPage(candidates); + inOrder.verify(pageMemberRepository).setPageMembersToNewPage(newId, ids); + inOrder.verify(pageRelationRepository).updateCompactionBucketRelations(ids, newId); + inOrder.verify(pageDeletionTimeSetter).setDeleteTimeOfFragment(ids); + inOrder.verifyNoMoreInteractions(); + } + + private static Set createCompactionCandidates() { + return new HashSet<>(Set.of( + new CompactionCandidate(1L, 5, 2L, 1L, "http://example.com"), + new CompactionCandidate(2L, 5, 3L, 1L, "http://example.com"), + new CompactionCandidate(3L, 5, 4L, 1L, "http://example.com") + )); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTaskTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTaskTest.java new file mode 100644 index 000000000..483e6a62b --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/PageDeletionTaskTest.java @@ -0,0 +1,26 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.pagination.repositories.PageRepository; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class PageDeletionTaskTest { + @Mock + private PageRepository pageRepository; + @InjectMocks + private PageDeletionTask pageDeletionTask; + + @Test + void when_FragmentHasDeleteTimeEarlierThanCurrentTime_then_ItIsDeletedAndEventIsSent() { + pageDeletionTask.execute(null, null); + + verify(pageRepository).deleteOutdatedFragments(any()); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitionerTest.java b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitionerTest.java new file mode 100644 index 000000000..2a2bdf708 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-compaction/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/compaction/batch/ViewCapacityPartitionerTest.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.compaction.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.entities.ViewCapacity; +import be.vlaanderen.informatievlaanderen.ldes.server.compaction.domain.repository.ViewCollection; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.item.ExecutionContext; + +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ViewCapacityPartitionerTest { + private static final String COLLECTION = "collection"; + private static final ViewName VIEW_A = new ViewName(COLLECTION, "viewA"); + private static final ViewName VIEW_B = new ViewName(COLLECTION, "viewB"); + private static final ViewName VIEW_C = new ViewName(COLLECTION, "viewC"); + @Mock + private ViewCollection viewCollection; + @InjectMocks + private ViewCapacityPartitioner viewCapacityPartitioner; + + @Test + void given_NonEmptyCollection_testPartitioning() { + int capacityA = 20; + int capacityB = 40; + int capacityC = 60; + + + when(viewCollection.getAllViewCapacities()).thenReturn(Set.of( + new ViewCapacity(VIEW_A, capacityA), + new ViewCapacity(VIEW_B, capacityB), + new ViewCapacity(VIEW_C, capacityC) + )); + + final Map result = viewCapacityPartitioner.partition(0); + + assertThat(result) + .hasSize(3) + .containsAllEntriesOf(Map.of( + "view:%s".formatted(VIEW_A.asString()), + new ExecutionContext(Map.of("viewName", VIEW_A.asString(), "capacityPerPage", capacityA)), + "view:%s".formatted(VIEW_B.asString()), + new ExecutionContext(Map.of("viewName", VIEW_B.asString(), "capacityPerPage", capacityB)), + "view:%s".formatted(VIEW_C.asString()), + new ExecutionContext(Map.of("viewName", VIEW_C.asString(), "capacityPerPage", capacityC)) + )); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml b/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml index 7d0336c06..390b269ac 100644 --- a/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml +++ b/ldes-server-maintenance/ldes-server-maintenance-common/pom.xml @@ -11,6 +11,40 @@ ldes-server-maintenance-common + + + org.springframework.boot + spring-boot-starter-batch + + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-domain + + + + org.junit.jupiter + junit-jupiter + + + org.assertj + assertj-core + + + org.springframework.boot + spring-boot-starter-test + + + org.springframework.batch + spring-batch-test + test + + + com.h2database + h2 + test + + + ${project.artifactId} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceFlows.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceFlows.java new file mode 100644 index 000000000..dbdfcbcf7 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceFlows.java @@ -0,0 +1,50 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.valueobjects.DecidedFlowExecutionStatus; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.job.flow.Flow; +import org.springframework.batch.core.job.flow.JobExecutionDecider; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MaintenanceFlows { + public static final String MAINTENANCE_JOB = "maintenance"; + + @Bean + public FlowJobBuilder maintenanceJobBuilder(JobRepository jobRepository, + Flow viewRetentionFlow, + Flow eventSourceRetentionFlow) { + return new JobBuilder(MAINTENANCE_JOB, jobRepository) + .start(viewRetentionFlow) + .next(eventSourceRetentionFlow) + .end(); + } + + @Bean + public Flow viewRetentionFlow(@Qualifier("viewRetentionStep") Step viewRetentionStep, + @Qualifier("compactionStep") Step compactionStep, + @Qualifier("deletionStep") Step deletionStep, + @Qualifier("viewRetentionExecutionDecider") JobExecutionDecider decider) { + return new FlowBuilder("viewRetentionFlow") + .start(decider).on(DecidedFlowExecutionStatus.SKIP.pattern()).end() + .from(decider).on(DecidedFlowExecutionStatus.CONTINUE.pattern()).to(viewRetentionStep) + .from(viewRetentionStep).on("*").to(compactionStep) + .from(compactionStep).on("*").to(deletionStep) + .end(); + } + + @Bean + public Flow eventSourceRetentionFlow(@Qualifier("eventSourceRetentionStep") Step step, + @Qualifier("eventSourceRetentionExecutionDecider") JobExecutionDecider decider) { + return new FlowBuilder("eventSourceRetentionFlow") + .start(decider).on(DecidedFlowExecutionStatus.SKIP.pattern()).end() + .from(decider).on(DecidedFlowExecutionStatus.CONTINUE.pattern()).to(step) + .end(); + } +} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceService.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceService.java new file mode 100644 index 000000000..819a648c1 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceService.java @@ -0,0 +1,50 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.exceptions.MaintenanceJobException; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecutionException; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConfig.MAINTENANCE_CRON_KEY; +import static be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch.MaintenanceFlows.MAINTENANCE_JOB; + +@Service +@EnableScheduling +public class MaintenanceService { + private final JobLauncher jobLauncher; + private final JobExplorer jobExplorer; + private final Job maintenanceJob; + + public MaintenanceService(JobLauncher jobLauncher, + JobExplorer jobExplorer, + FlowJobBuilder maintenanceJobBuilder) { + this.jobLauncher = jobLauncher; + this.jobExplorer = jobExplorer; + this.maintenanceJob = maintenanceJobBuilder.build(); + } + + @Scheduled(cron = MAINTENANCE_CRON_KEY) + public void scheduleMaintenanceJob() { + try { + if (hasNoJobsRunning()) { + jobLauncher.run(maintenanceJob, new JobParametersBuilder() + .addLocalDateTime("triggered", LocalDateTime.now()) + .toJobParameters()); + } + } catch (JobExecutionException e) { + throw new MaintenanceJobException(e); + } + } + + private boolean hasNoJobsRunning() { + return jobExplorer.findRunningJobExecutions(MAINTENANCE_JOB).isEmpty(); + } +} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/exceptions/MaintenanceJobException.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/exceptions/MaintenanceJobException.java new file mode 100644 index 000000000..906c65edd --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/exceptions/MaintenanceJobException.java @@ -0,0 +1,9 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.exceptions; + +import org.springframework.batch.core.JobExecutionException; + +public class MaintenanceJobException extends RuntimeException { + public MaintenanceJobException(JobExecutionException cause) { + super(cause); + } +} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/services/RetentionPolicyEmptinessChecker.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/services/RetentionPolicyEmptinessChecker.java new file mode 100644 index 000000000..98fb21b40 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/services/RetentionPolicyEmptinessChecker.java @@ -0,0 +1,5 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services; + +public interface RetentionPolicyEmptinessChecker { + boolean isEmpty(); +} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/valueobjects/DecidedFlowExecutionStatus.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/valueobjects/DecidedFlowExecutionStatus.java new file mode 100644 index 000000000..6d47b742c --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/valueobjects/DecidedFlowExecutionStatus.java @@ -0,0 +1,24 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.valueobjects; + +import org.springframework.batch.core.job.flow.FlowExecutionStatus; + +public enum DecidedFlowExecutionStatus { + SKIP("SKIP"), + CONTINUE("CONTINUE"); + + private final String pattern; + private final FlowExecutionStatus status; + + DecidedFlowExecutionStatus(String pattern) { + this.pattern = pattern; + this.status = new FlowExecutionStatus(pattern); + } + + public String pattern() { + return pattern; + } + + public FlowExecutionStatus status() { + return status; + } +} diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/module-info.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/module-info.java new file mode 100644 index 000000000..00b049824 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/main/java/module-info.java @@ -0,0 +1,8 @@ +open module ldes.server.maintenance.common { + requires spring.context; + requires spring.batch.core; + requires ldes.domain; + requires spring.beans; + exports be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services; + exports be.vlaanderen.informatievlaanderen.ldes.server.maintenance.valueobjects; +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceJobDefinitionTest.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceJobDefinitionTest.java new file mode 100644 index 000000000..8516a5b26 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceJobDefinitionTest.java @@ -0,0 +1,161 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.batch.core.*; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.job.flow.FlowExecutionStatus; +import org.springframework.batch.core.job.flow.JobExecutionDecider; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.batch.test.context.SpringBatchTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.transaction.PlatformTransactionManager; + +import java.time.LocalDateTime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(SpringExtension.class) +@SpringBatchTest +@SpringBootTest(classes = {MaintenanceFlows.class}) +@EnableAutoConfiguration +@Import(MaintenanceJobDefinitionTest.TestSteps.class) +class MaintenanceJobDefinitionTest { + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + @Autowired + private FlowJobBuilder maintenanceJobBuilder; + @SpyBean(name = "viewRetentionStep") + private Step viewRetentionStep; + @SpyBean(name = "compactionStep") + private Step compactionStep; + @SpyBean(name = "deletionStep") + private Step deletionStep; + @SpyBean(name = "eventSourceRetentionStep") + private Step eventSourceRetentionStep; + + @MockBean(name = "viewRetentionExecutionDecider") + private JobExecutionDecider viewRetentionExecutionDecider; + @MockBean(name = "eventSourceRetentionExecutionDecider") + private JobExecutionDecider eventSourceRetentionExecutionDecider; + + @BeforeEach + void setUp() { + jobLauncherTestUtils.setJob(maintenanceJobBuilder.build()); + } + + @Test + void given_EventSourceRetentionShouldBeSkipped_when_Execute_then_VerifyNoEventSourceRetentionStepInteraction() throws Exception { + when(viewRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("CONTINUE")); + when(eventSourceRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("SKIP")); + final JobParameters jobParameters = new JobParametersBuilder() + .addLocalDateTime("triggered", LocalDateTime.now()) + .toJobParameters(); + + final JobExecution result = jobLauncherTestUtils.launchJob(jobParameters); + + assertThat(result.getExitStatus().getExitCode()).isEqualTo(ExitStatus.COMPLETED.getExitCode()); + verify(viewRetentionStep).execute(any()); + verify(compactionStep).execute(any()); + verify(deletionStep).execute(any()); + verify(eventSourceRetentionStep, never()).execute(any()); + } + + @Test + void given_NothingShouldBeSkipped_when_Execute_then_VerifyAllStepsInteraction() throws Exception { + when(viewRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("CONTINUE")); + when(eventSourceRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("CONTINUE")); + final JobParameters jobParameters = new JobParametersBuilder() + .addLocalDateTime("triggered", LocalDateTime.now()) + .toJobParameters(); + + final JobExecution result = jobLauncherTestUtils.launchJob(jobParameters); + + assertThat(result.getExitStatus().getExitCode()).isEqualTo(ExitStatus.COMPLETED.getExitCode()); + verify(viewRetentionStep).execute(any()); + verify(compactionStep).execute(any()); + verify(deletionStep).execute(any()); + verify(eventSourceRetentionStep).execute(any()); + } + + @Test + void given_AllShouldBeSkipped_when_Execute_then_VerifyNoStepsInteraction() throws Exception { + when(viewRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("SKIP")); + when(eventSourceRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("SKIP")); + final JobParameters jobParameters = new JobParametersBuilder() + .addLocalDateTime("triggered", LocalDateTime.now()) + .toJobParameters(); + + final JobExecution result = jobLauncherTestUtils.launchJob(jobParameters); + + assertThat(result.getExitStatus().getExitCode()).isEqualTo(ExitStatus.NOOP.getExitCode()); + verify(viewRetentionStep, never()).execute(any()); + verify(compactionStep, never()).execute(any()); + verify(deletionStep, never()).execute(any()); + verify(eventSourceRetentionStep, never()).execute(any()); + } + + @Test + void given_ViewRetentionShouldBeSkipped_when_Execute_then_VerifyOnlyEventSourceRetentionStepInteraction() throws Exception { + when(viewRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("SKIP")); + when(eventSourceRetentionExecutionDecider.decide(any(), any())).thenReturn(new FlowExecutionStatus("CONTINUE")); + final JobParameters jobParameters = new JobParametersBuilder() + .addLocalDateTime("triggered", LocalDateTime.now()) + .toJobParameters(); + + final JobExecution result = jobLauncherTestUtils.launchJob(jobParameters); + + assertThat(result.getExitStatus().getExitCode()).isEqualTo(ExitStatus.COMPLETED.getExitCode()); + verify(viewRetentionStep, never()).execute(any()); + verify(compactionStep, never()).execute(any()); + verify(deletionStep, never()).execute(any()); + verify(eventSourceRetentionStep).execute(any()); + } + + @TestConfiguration + static class TestSteps { + private final Tasklet tasklet = (contribution, chunkContext) -> RepeatStatus.FINISHED; + + @Bean(name = "viewRetentionStep") + public Step viewRetentionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("viewRetentionStep", jobRepository) + .tasklet(tasklet, transactionManager) + .build(); + } + + @Bean(name = "compactionStep") + public Step compactionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("compactionStep", jobRepository) + .tasklet(tasklet, transactionManager) + .build(); + } + @Bean(name = "deletionStep") + public Step deletionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("deletionStep", jobRepository) + .tasklet(tasklet, transactionManager) + .build(); + } + @Bean(name = "eventSourceRetentionStep") + public Step eventSourceRetentionStep(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new StepBuilder("eventSourceRetentionStep", jobRepository) + .tasklet(tasklet, transactionManager) + .build(); + } + } + + +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceServiceTest.java b/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceServiceTest.java new file mode 100644 index 000000000..6685c2a18 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-maintenance-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/batch/MaintenanceServiceTest.java @@ -0,0 +1,83 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.exceptions.MaintenanceJobException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.*; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.*; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.job.builder.FlowJobBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; +import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; +import org.springframework.batch.core.repository.JobRestartException; + +import java.util.Set; +import java.util.stream.Stream; + +import static be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch.MaintenanceFlows.MAINTENANCE_JOB; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class MaintenanceServiceTest { + @Mock + private FlowJobBuilder flowJobBuilder; + @Mock + private JobLauncher jobLauncher; + @Mock + private Job maintenanceJob; + @Mock(answer = Answers.RETURNS_MOCKS) + private JobExplorer jobExplorer; + @InjectMocks + private MaintenanceService maintenanceService; + @Captor + private ArgumentCaptor jobParametersCaptor; + + + @Test + void given_NoRunningJobs_when_ScheduleJob_then_JobIsScheduled() throws JobExecutionException { + maintenanceService.scheduleMaintenanceJob(); + + verify(jobLauncher).run(any(), jobParametersCaptor.capture()); + + } + + @Test + void given_JobsIsRunning_when_ScheduleJob_then_JobIsNotScheduled() { + when(jobExplorer.findRunningJobExecutions(MAINTENANCE_JOB)).thenReturn(Set.of(new JobExecution(1L))); + + maintenanceService.scheduleMaintenanceJob(); + + verifyNoInteractions(jobLauncher); + assertThat(jobParametersCaptor.getAllValues()).isEmpty(); + } + + @ParameterizedTest + @MethodSource("jobExecutionExceptions") + void when_JobExecutionExceptionIsThrownDuringLaunch_then_ThrowMaintenanceException(Class exceptionClass) throws JobExecutionException { + when(jobLauncher.run(any(), any())).thenThrow(exceptionClass); + + assertThatThrownBy(maintenanceService::scheduleMaintenanceJob) + .isInstanceOf(MaintenanceJobException.class) + .hasCauseInstanceOf(exceptionClass); + verify(jobLauncher).run(any(), jobParametersCaptor.capture()); + assertThat(jobParametersCaptor.getValue()) + .extracting(jobParams -> jobParams.getLocalDateTime("triggered")) + .isNotNull(); + } + + static Stream> jobExecutionExceptions() { + return Stream.of( + JobInstanceAlreadyCompleteException.class, + JobExecutionAlreadyRunningException.class, + JobParametersInvalidException.class, + JobRestartException.class + ); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/pom.xml b/ldes-server-maintenance/ldes-server-retention/pom.xml index 74725e7c3..8f149cc97 100644 --- a/ldes-server-maintenance/ldes-server-retention/pom.xml +++ b/ldes-server-maintenance/ldes-server-retention/pom.xml @@ -24,6 +24,10 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-domain + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-maintenance-common + org.awaitility awaitility diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDecider.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDecider.java new file mode 100644 index 000000000..4b14e977f --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDecider.java @@ -0,0 +1,25 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.decider; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services.RetentionPolicyEmptinessChecker; +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.valueobjects.DecidedFlowExecutionStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.job.flow.FlowExecutionStatus; +import org.springframework.batch.core.job.flow.JobExecutionDecider; + +import java.util.List; + +public class RetentionExecutionDecider implements JobExecutionDecider { + private final List retentionEmptinessCheckers; + + public RetentionExecutionDecider(List retentionEmptinessCheckers) { + this.retentionEmptinessCheckers = retentionEmptinessCheckers; + } + + @Override + public FlowExecutionStatus decide(JobExecution jobExecution, StepExecution stepExecution) { + return retentionEmptinessCheckers.stream().allMatch(RetentionPolicyEmptinessChecker::isEmpty) + ? DecidedFlowExecutionStatus.SKIP.status() + : DecidedFlowExecutionStatus.CONTINUE.status(); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciders.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciders.java new file mode 100644 index 000000000..0bfd95774 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciders.java @@ -0,0 +1,20 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.decider; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services.RetentionPolicyEmptinessChecker; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +@Configuration +public class RetentionExecutionDeciders { + @Bean + public RetentionExecutionDecider eventSourceRetentionExecutionDecider(List emptinessCheckers) { + return new RetentionExecutionDecider(emptinessCheckers); + } + + @Bean + public RetentionExecutionDecider viewRetentionExecutionDecider(RetentionPolicyEmptinessChecker viewRetentionPolicyCollection) { + return new RetentionExecutionDecider(List.of(viewRetentionPolicyCollection)); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPartitionerConfig.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPartitionerConfig.java new file mode 100644 index 000000000..850eea83e --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPartitionerConfig.java @@ -0,0 +1,20 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.partitioner; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.EventSourceRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.ViewRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies.RetentionPolicyCollection; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RetentionPartitionerConfig { + @Bean + public RetentionPolicyPartitioner viewRetentionPolicyPartitioner(RetentionPolicyCollection viewRetentionPolicyCollection) { + return new RetentionPolicyPartitioner(viewRetentionPolicyCollection); + } + + @Bean + public RetentionPolicyPartitioner eventSourceRetentionPolicyPartitioner(RetentionPolicyCollection eventSourceRetentionPolicyCollection) { + return new RetentionPolicyPartitioner(eventSourceRetentionPolicyCollection); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitioner.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitioner.java new file mode 100644 index 000000000..1c3cea436 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitioner.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.partitioner; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.RetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies.RetentionPolicyCollection; +import org.springframework.batch.core.partition.support.Partitioner; +import org.springframework.batch.item.ExecutionContext; + +import java.util.Map; +import java.util.stream.Collectors; + +public class RetentionPolicyPartitioner implements Partitioner { + private final RetentionPolicyCollection retentionPolicyCollection; + + public RetentionPolicyPartitioner(RetentionPolicyCollection retentionPolicyCollection) { + this.retentionPolicyCollection = retentionPolicyCollection; + } + + @Override + public Map partition(int gridSize) { + return retentionPolicyCollection.getRetentionPolicies().stream().collect( + Collectors.toMap( + retentionPolicy -> "retention:%s".formatted(retentionPolicy.getName()), + retentionPolicy -> new ExecutionContext(Map.of( + "name", retentionPolicy.getName(), + "retentionPolicy", retentionPolicy.retentionPolicy() + )) + ) + ); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/EventSourceRetentionTask.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/EventSourceRetentionTask.java new file mode 100644 index 000000000..c2b92f7ee --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/EventSourceRetentionTask.java @@ -0,0 +1,46 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberProperties; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.execution.MemberRemover; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component +public class EventSourceRetentionTask extends RetentionTask { + private static final Logger log = LoggerFactory.getLogger(EventSourceRetentionTask.class); + private final MemberPropertiesRepository memberPropertiesRepository; + private final MemberRemover memberRemover; + + public EventSourceRetentionTask(MemberPropertiesRepository memberPropertiesRepository, MemberRemover memberRemover) { + this.memberPropertiesRepository = memberPropertiesRepository; + this.memberRemover = memberRemover; + } + + protected void removeMembersThatMatchRetentionPolicies(String collectionName, RetentionPolicy retentionPolicy) { + log.atDebug().log("Start retention for event source of collection {}", collectionName); + final Stream memberPropertiesStream = switch (retentionPolicy.getType()) { + case TIME_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, + (TimeBasedRetentionPolicy) retentionPolicy); + case VERSION_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, + (VersionBasedRetentionPolicy) retentionPolicy); + case TIME_AND_VERSION_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, + (TimeAndVersionBasedRetentionPolicy) retentionPolicy); + }; + + Map> areMembersRemoveableMap = memberPropertiesStream.collect(Collectors.partitioningBy(memberProperties -> !memberProperties.isInView())); + memberRemover.deleteMembers(areMembersRemoveableMap.get(true)); + memberRemover.removeMembersFromEventSource(areMembersRemoveableMap.get(false)); + log.atDebug().log("Finished retention for event source of collection {}", collectionName); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/RetentionTask.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/RetentionTask.java new file mode 100644 index 000000000..50a978804 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/RetentionTask.java @@ -0,0 +1,23 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +public abstract class RetentionTask implements Tasklet { + + @Override + public final RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + final ExecutionContext executionContext = chunkContext.getStepContext().getStepExecution().getExecutionContext(); + if(executionContext.get("retentionPolicy") instanceof RetentionPolicy retentionPolicy) { + final String name = executionContext.getString("name"); + removeMembersThatMatchRetentionPolicies(name, retentionPolicy); + } + return RepeatStatus.FINISHED; + } + + protected abstract void removeMembersThatMatchRetentionPolicies(String name, RetentionPolicy retentionPolicy); +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/ViewRetentionTask.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/ViewRetentionTask.java new file mode 100644 index 000000000..3879a7889 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/retentiontasklet/ViewRetentionTask.java @@ -0,0 +1,42 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class ViewRetentionTask extends RetentionTask { + private static final Logger log = LoggerFactory.getLogger(ViewRetentionTask.class); + private final MemberPropertiesRepository memberPropertiesRepository; + private final PageMemberRepository pageMemberRepository; + + public ViewRetentionTask(MemberPropertiesRepository memberPropertiesRepository, PageMemberRepository pageMemberRepository) { + this.memberPropertiesRepository = memberPropertiesRepository; + this.pageMemberRepository = pageMemberRepository; + } + + @Override + protected void removeMembersThatMatchRetentionPolicies(String name, RetentionPolicy retentionPolicy) { + final ViewName viewName = ViewName.fromString(name); + log.atDebug().log("Start retention for view: {}", name); + List expiredMemberIds = switch (retentionPolicy.getType()) { + case TIME_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, + (TimeBasedRetentionPolicy) retentionPolicy); + case VERSION_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, + (VersionBasedRetentionPolicy) retentionPolicy); + case TIME_AND_VERSION_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, + (TimeAndVersionBasedRetentionPolicy) retentionPolicy); + }; + pageMemberRepository.deleteByViewNameAndMembersIds(viewName, expiredMemberIds); + log.atDebug().log("Finished retention for view: {}", viewName.asString()); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/stepdefinition/RetentionStepDefinitions.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/stepdefinition/RetentionStepDefinitions.java new file mode 100644 index 000000000..2a6a8b39f --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/stepdefinition/RetentionStepDefinitions.java @@ -0,0 +1,65 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.stepdefinition; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.partitioner.RetentionPolicyPartitioner; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet.EventSourceRetentionTask; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet.ViewRetentionTask; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +public class RetentionStepDefinitions { + public static final String VIEW_RETENTION_STEP = "viewRetentionStep"; + public static final String EVENT_SOURCE_RETENTION_STEP = "eventSourceRetentionStep"; + private static final String SINGLE_VIEW_RETENTION_STEP = "singleViewRetentionStep"; + private static final String SINGLE_COLLECTION_RETENTION_STEP = "singleEventSourceRetentionStep"; + private static final String VIEW_PARTITIONER = "viewRetentionPartitioner"; + private static final String COLLECTION_PARTITIONER = "eventSourceRetentionPartitioner"; + + @Bean + public Step viewRetentionStep(JobRepository jobRepository, + RetentionPolicyPartitioner viewRetentionPolicyPartitioner, + ViewRetentionTask viewRetentionTask, + TaskExecutor viewRetentionExecutor, + PlatformTransactionManager transactionManager) { + return new StepBuilder(VIEW_RETENTION_STEP, jobRepository) + .partitioner(VIEW_PARTITIONER, viewRetentionPolicyPartitioner) + .step(new StepBuilder(SINGLE_VIEW_RETENTION_STEP, jobRepository) + .tasklet(viewRetentionTask, transactionManager) + .build() + ) + .taskExecutor(viewRetentionExecutor) + .build(); + } + + @Bean + public TaskExecutor viewRetentionExecutor() { + return new SimpleAsyncTaskExecutor("view_retention_batch"); + } + + @Bean + public Step eventSourceRetentionStep(JobRepository jobRepository, + RetentionPolicyPartitioner eventSourceRetentionPolicyPartitioner, + EventSourceRetentionTask eventSourceRetentionTask, + TaskExecutor eventSourceRetentionExecutor, + PlatformTransactionManager transactionManager) { + return new StepBuilder(EVENT_SOURCE_RETENTION_STEP, jobRepository) + .partitioner(COLLECTION_PARTITIONER, eventSourceRetentionPolicyPartitioner) + .step(new StepBuilder(SINGLE_COLLECTION_RETENTION_STEP, jobRepository) + .tasklet(eventSourceRetentionTask, transactionManager) + .build() + ) + .taskExecutor(eventSourceRetentionExecutor) + .build(); + } + + @Bean + public TaskExecutor eventSourceRetentionExecutor() { + return new SimpleAsyncTaskExecutor("event_source_retention_batch"); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java deleted file mode 100644 index 091a65ed9..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/config/AsyncConfig.java +++ /dev/null @@ -1,9 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableAsync; - -@Configuration -@EnableAsync -public class AsyncConfig { -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProvider.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProvider.java new file mode 100644 index 000000000..505146c93 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProvider.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; + +import java.util.Objects; + +public record EventSourceRetentionPolicyProvider(String collectionName, + RetentionPolicy retentionPolicy) implements RetentionPolicyProvider { + + @Override + public String getName() { + return collectionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof EventSourceRetentionPolicyProvider that)) return false; + return Objects.equals(collectionName, that.collectionName); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + Objects.hashCode(collectionName); + return result; + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/RetentionPolicyProvider.java similarity index 51% rename from ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java rename to ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/RetentionPolicyProvider.java index c75809a60..12db70d74 100644 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollection.java +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/RetentionPolicyProvider.java @@ -1,10 +1,9 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; +package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import java.util.Map; +public interface RetentionPolicyProvider { + String getName(); + RetentionPolicy retentionPolicy(); -public interface DeletionPolicyCollection { - Map getEventSourceRetentionPolicyMap(); - boolean isEmpty(); } diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProvider.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProvider.java new file mode 100644 index 000000000..cb62af2b6 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProvider.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; + +import java.util.Objects; + +public record ViewRetentionPolicyProvider(ViewName viewName, RetentionPolicy retentionPolicy) implements RetentionPolicyProvider { + + @Override + public String getName() { + return viewName().asString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ViewRetentionPolicyProvider that)) return false; + return Objects.equals(viewName, that.viewName); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + Objects.hashCode(viewName); + return result; + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java deleted file mode 100644 index e12088f8a..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.DeletionPolicyChangedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import org.apache.jena.rdf.model.Model; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Component -public class DeletionPolicyCollectionImpl implements DeletionPolicyCollection { - private final RetentionPolicyFactory retentionPolicyFactory; - private final Map eventSourceRetentionPolicyMap; - - public DeletionPolicyCollectionImpl(RetentionPolicyFactory retentionPolicyFactory) { - this.retentionPolicyFactory = retentionPolicyFactory; - this.eventSourceRetentionPolicyMap = new HashMap<>(); - } - @Override - public Map getEventSourceRetentionPolicyMap() { - return Map.copyOf(eventSourceRetentionPolicyMap); - } - - @Override - public boolean isEmpty() { - return eventSourceRetentionPolicyMap.isEmpty(); - } - - @EventListener - public void handleDeletionPolicyChangedEvent(DeletionPolicyChangedEvent event) { - eventSourceRetentionPolicyMap.remove(event.collectionName()); - addToMap(event.collectionName(), event.retentionPolicies()); - } - - @EventListener - public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { - eventSourceRetentionPolicyMap.remove(event.collectionName()); - } - - private void addToMap(String collectionName, List retentionPolicyModels) { - if (!retentionPolicyModels.isEmpty()) { - retentionPolicyFactory - .extractRetentionPolicy(retentionPolicyModels) - .ifPresent(policy -> eventSourceRetentionPolicyMap.put(collectionName, policy)); - } - } -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java deleted file mode 100644 index 5ff532ec0..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollection.java +++ /dev/null @@ -1,12 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.spi.RetentionPolicyEmptinessChecker; - -import java.util.Map; - -public interface RetentionPolicyCollection extends RetentionPolicyEmptinessChecker { - - Map getRetentionPolicyMap(); -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java deleted file mode 100644 index 6537db417..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.Map; - -@Component -public class RetentionPolicyCollectionImpl implements RetentionPolicyCollection { - - private final Map retentionPolicyMap; - private final RetentionPolicyFactory retentionPolicyFactory; - - public RetentionPolicyCollectionImpl(RetentionPolicyFactory retentionPolicyFactory) { - this.retentionPolicyFactory = retentionPolicyFactory; - this.retentionPolicyMap = new HashMap<>(); - } - - @EventListener - public void handleViewAddedEvent(ViewAddedEvent event) { - addToMap(event.getViewName(), event.viewSpecification()); - } - - @EventListener - public void handleViewInitializationEvent(ViewInitializationEvent event) { - addToMap(event.getViewName(), event.viewSpecification()); - } - - @EventListener - public void handleViewDeletedEvent(ViewDeletedEvent event) { - retentionPolicyMap.remove(event.getViewName()); - } - - @EventListener - public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { - retentionPolicyMap.keySet().stream() - .filter(viewName -> viewName.getCollectionName().equals(event.collectionName())) - .toList() - .forEach(retentionPolicyMap::remove); - } - - @Override - public Map getRetentionPolicyMap() { - return Map.copyOf(retentionPolicyMap); - } - - @Override - public boolean isEmpty() { - return retentionPolicyMap.isEmpty(); - } - - private void addToMap(ViewName viewName, ViewSpecification viewSpecification) { - retentionPolicyFactory - .extractRetentionPolicy(viewSpecification) - .ifPresent(policy -> retentionPolicyMap.put(viewName, policy)); - } - -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/EventSourceRetentionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/EventSourceRetentionPolicyCollection.java new file mode 100644 index 000000000..8c14f4bd1 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/EventSourceRetentionPolicyCollection.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.DeletionPolicyChangedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.EventSourceRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; +import org.apache.jena.rdf.model.Model; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class EventSourceRetentionPolicyCollection implements RetentionPolicyCollection { + private final RetentionPolicyFactory retentionPolicyFactory; + private final Set retentionPolicies; + + public EventSourceRetentionPolicyCollection(RetentionPolicyFactory retentionPolicyFactory) { + this.retentionPolicyFactory = retentionPolicyFactory; + retentionPolicies = new HashSet<>(); + } + + @Override + public Set getRetentionPolicies() { + return Set.copyOf(retentionPolicies); + } + + @Override + public boolean isEmpty() { + return retentionPolicies.isEmpty(); + } + + @EventListener + public void handleDeletionPolicyChangedEvent(DeletionPolicyChangedEvent event) { + removeFromCollection(event.collectionName()); + addToCollection(event.collectionName(), event.retentionPolicies()); + } + + @EventListener + public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { + removeFromCollection(event.collectionName()); + } + + private void removeFromCollection(String collectionName) { + retentionPolicies.removeIf(policy -> policy.collectionName().equals(collectionName)); + } + + private void addToCollection(String collectionName, List retentionPolicyModels) { + if (!retentionPolicyModels.isEmpty()) { + retentionPolicyFactory + .extractRetentionPolicy(retentionPolicyModels) + .map(retentionPolicy -> new EventSourceRetentionPolicyProvider(collectionName, retentionPolicy)) + .ifPresent(retentionPolicies::add); + } + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/RetentionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/RetentionPolicyCollection.java new file mode 100644 index 000000000..dc8a0d6bc --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/RetentionPolicyCollection.java @@ -0,0 +1,9 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services.RetentionPolicyEmptinessChecker; + +import java.util.Set; + +public interface RetentionPolicyCollection extends RetentionPolicyEmptinessChecker { + Set getRetentionPolicies(); +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/ViewRetentionPolicyCollection.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/ViewRetentionPolicyCollection.java new file mode 100644 index 000000000..98d3b9453 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/retentionpolicies/ViewRetentionPolicyCollection.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.*; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.ViewRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.Set; + +@Component +public class ViewRetentionPolicyCollection implements RetentionPolicyCollection { + + private final Set retentionPolicies; + private final RetentionPolicyFactory retentionPolicyFactory; + + public ViewRetentionPolicyCollection(RetentionPolicyFactory retentionPolicyFactory) { + this.retentionPolicyFactory = retentionPolicyFactory; + this.retentionPolicies = new HashSet<>(); + } + + @EventListener(classes = {ViewInitializationEvent.class, ViewAddedEvent.class}) + public void handleViewAddedEvent(ViewSupplier event) { + addToCollection(event.viewSpecification()); + } + + @EventListener + public void handleViewDeletedEvent(ViewDeletedEvent event) { + retentionPolicies.removeIf(retentionPolicy -> retentionPolicy.viewName().equals(event.getViewName())); + } + + @EventListener + public void handleEventStreamDeletedEvent(EventStreamDeletedEvent event) { + retentionPolicies.removeIf(retentionPolicy -> retentionPolicy.viewName().getCollectionName().equals(event.collectionName())); + } + + @Override + public Set getRetentionPolicies() { + return Set.copyOf(retentionPolicies); + } + + @Override + public boolean isEmpty() { + return retentionPolicies.isEmpty(); + } + + private void addToCollection(ViewSpecification viewSpecification) { + retentionPolicyFactory + .extractRetentionPolicy(viewSpecification) + .map(retentionPolicy -> new ViewRetentionPolicyProvider(viewSpecification.getName(), retentionPolicy)) + .ifPresent(retentionPolicies::add); + } + +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java index 3f31d6861..185d9022f 100644 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/definition/RetentionPolicy.java @@ -1,6 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition; -public interface RetentionPolicy { +import java.io.Serializable; + +public interface RetentionPolicy extends Serializable { RetentionPolicyType getType(); diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java deleted file mode 100644 index 1d19e3fdc..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionService.java +++ /dev/null @@ -1,92 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.execution; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberProperties; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.DeletionPolicyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.RetentionPolicyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConfig.RETENTION_CRON_KEY; - -@Service -@EnableScheduling -public class RetentionService { - - private static final Logger log = LoggerFactory.getLogger(RetentionService.class); - - private final MemberPropertiesRepository memberPropertiesRepository; - private final PageMemberRepository pageMemberRepository; - private final MemberRemover memberRemover; - private final RetentionPolicyCollection retentionPolicyCollection; - private final DeletionPolicyCollection deletionPolicyCollection; - - public RetentionService(MemberPropertiesRepository memberPropertiesRepository, PageMemberRepository pageMemberRepository, MemberRemover memberRemover, - RetentionPolicyCollection retentionPolicyCollection, DeletionPolicyCollection deletionPolicyCollection) { - this.memberPropertiesRepository = memberPropertiesRepository; - this.pageMemberRepository = pageMemberRepository; - this.memberRemover = memberRemover; - this.retentionPolicyCollection = retentionPolicyCollection; - this.deletionPolicyCollection = deletionPolicyCollection; - } - - @SuppressWarnings("java:S6857") - @Scheduled(cron = RETENTION_CRON_KEY) - public void executeRetentionPolicies() { - if(retentionPolicyCollection.isEmpty() && deletionPolicyCollection.isEmpty()) { - log.atDebug().log("Retention skipped: no retention policies found"); - return; - } - log.atDebug().log("Start retention"); - retentionPolicyCollection - .getRetentionPolicyMap() - .forEach(this::removeMembersFromViewThatMatchRetentionPolicies); - deletionPolicyCollection - .getEventSourceRetentionPolicyMap() - .forEach(this::removeMembersFromEventSourceThatMatchRetentionPolicies); - log.atDebug().log("Finish retention"); - } - - private void removeMembersFromViewThatMatchRetentionPolicies(ViewName viewName, - RetentionPolicy retentionPolicy) { - List expiredMemberIds = switch (retentionPolicy.getType()) { - case TIME_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, - (TimeBasedRetentionPolicy) retentionPolicy); - case VERSION_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, - (VersionBasedRetentionPolicy) retentionPolicy); - case TIME_AND_VERSION_BASED -> memberPropertiesRepository.findExpiredMembers(viewName, - (TimeAndVersionBasedRetentionPolicy) retentionPolicy); - }; - pageMemberRepository.deleteByViewNameAndMembersIds(viewName, expiredMemberIds); - } - - private void removeMembersFromEventSourceThatMatchRetentionPolicies(String collectionName, - RetentionPolicy retentionPolicy) { - final Stream memberPropertiesStream = switch (retentionPolicy.getType()) { - case TIME_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, - (TimeBasedRetentionPolicy) retentionPolicy); - case VERSION_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, - (VersionBasedRetentionPolicy) retentionPolicy); - case TIME_AND_VERSION_BASED -> memberPropertiesRepository.retrieveExpiredMembers(collectionName, - (TimeAndVersionBasedRetentionPolicy) retentionPolicy); - }; - - Map> areMembersRemoveableMap = memberPropertiesStream.collect(Collectors.partitioningBy(memberProperties -> !memberProperties.isInView())); - memberRemover.deleteMembers(areMembersRemoveableMap.get(true)); - memberRemover.removeMembersFromEventSource(areMembersRemoveableMap.get(false)); - } -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java deleted file mode 100644 index 53586388d..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/SchedulingConfig.java +++ /dev/null @@ -1,10 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.execution; - -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; - -@Configuration -@EnableScheduling -public class SchedulingConfig { - -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java deleted file mode 100644 index 9b9603197..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/spi/RetentionPolicyEmptinessChecker.java +++ /dev/null @@ -1,5 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.spi; - -public interface RetentionPolicyEmptinessChecker { - boolean isEmpty(); -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java index 659f8be2c..635cf440a 100644 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/module-info.java @@ -1,8 +1,12 @@ open module ldes.server.retention { requires ldes.domain; requires spring.context; + requires spring.batch.core; requires org.slf4j; requires org.apache.jena.core; requires org.apache.jena.arq; - exports be.vlaanderen.informatievlaanderen.ldes.server.retention.spi; + requires spring.batch.infrastructure; + requires ldes.server.maintenance.common; + requires spring.tx; + requires spring.core; } \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/EventSourceRetentionTaskTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/EventSourceRetentionTaskTest.java new file mode 100644 index 000000000..2cdf3ae24 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/EventSourceRetentionTaskTest.java @@ -0,0 +1,108 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet.EventSourceRetentionTask; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberProperties; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.execution.MemberRemover; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.scope.context.StepContext; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class EventSourceRetentionTaskTest { + private static final String COLLECTION = "collection"; + + @Mock + private StepExecution stepExecution; + @Mock + private ExecutionContext executionContext; + @Mock + private MemberPropertiesRepository memberPropertiesRepository; + @Mock + private MemberRemover memberRemover; + @InjectMocks + private EventSourceRetentionTask eventSourceRetentionTask; + + @BeforeEach + void setUp() { + when(stepExecution.getExecutionContext()).thenReturn(executionContext); + } + + @Test + void when_MembersOfFragmentMatchRetentionPoliciesOfEventSource_MembersAreRemovedFromTheEventSource() { + MemberProperties firstMember = createMemberProperties(1L, 0, true); + MemberProperties secondMember = createMemberProperties(2L, 1, false); + + var timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); + when(memberPropertiesRepository.retrieveExpiredMembers(COLLECTION, timeAndVersionBasedRetentionPolicy)) + .thenReturn(Stream.of(firstMember, secondMember)); + + when(executionContext.get("retentionPolicy")).thenReturn(timeAndVersionBasedRetentionPolicy); + when(executionContext.getString("name")).thenReturn(COLLECTION); + + final RepeatStatus returnedRepeatStatus = eventSourceRetentionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + + verify(memberRemover).removeMembersFromEventSource(List.of(firstMember)); + verify(memberRemover).deleteMembers(List.of(secondMember)); + assertThat(returnedRepeatStatus).isEqualTo(RepeatStatus.FINISHED); + } + + @Test + void given_NoRetentionPolicies_when_RetentionPoliciesExecuted_then_DeleteNoMembers() { + final RepeatStatus returnedRepeatStatus = eventSourceRetentionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + + verifyNoInteractions(memberRemover); + assertThat(returnedRepeatStatus).isEqualTo(RepeatStatus.FINISHED); + } + + @Test + void when_Execute_then_UseRightPolicy() { + final String collection1 = COLLECTION + "1"; + final String collection2 = COLLECTION + "2"; + final String collection3 = COLLECTION + "3"; + var timeBasedRetentionPolicy = new TimeBasedRetentionPolicy(Duration.ZERO); + var versionBasedRetentionPolicy = new VersionBasedRetentionPolicy(1); + var timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); + + when(executionContext.get("retentionPolicy")) + .thenReturn(timeBasedRetentionPolicy) + .thenReturn(versionBasedRetentionPolicy) + .thenReturn(timeAndVersionBasedRetentionPolicy); + when(executionContext.getString("name")) + .thenReturn(collection1) + .thenReturn(collection2) + .thenReturn(collection3); + + for (int i = 0; i < 3; i++) { + eventSourceRetentionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + } + + verify(memberPropertiesRepository).retrieveExpiredMembers(collection1, timeBasedRetentionPolicy); + verify(memberPropertiesRepository).retrieveExpiredMembers(collection2, versionBasedRetentionPolicy); + verify(memberPropertiesRepository).retrieveExpiredMembers(collection3, timeAndVersionBasedRetentionPolicy); + } + + private MemberProperties createMemberProperties(long memberId, int plusDays, boolean inView) { + return new MemberProperties(memberId, "coll", "http://ex.com", + LocalDateTime.now().plusDays(plusDays), true, inView); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/ViewRetentionTaskTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/ViewRetentionTaskTest.java new file mode 100644 index 000000000..c34713e57 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/ViewRetentionTaskTest.java @@ -0,0 +1,88 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.retentiontasklet.ViewRetentionTask; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.scope.context.StepContext; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.repeat.RepeatStatus; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ViewRetentionTaskTest { + private static final String COLLECTION = "collection"; + private static final ViewName VIEW_A = new ViewName(COLLECTION, "viewA"); + private static final ViewName VIEW_B = new ViewName(COLLECTION, "viewB"); + private static final ViewName VIEW_C = new ViewName(COLLECTION, "viewC"); + + @Mock + private MemberPropertiesRepository memberPropertiesRepository; + @Mock + private PageMemberRepository pageMemberRepository; + @Mock + private StepExecution stepExecution; + @Mock + private ExecutionContext executionContext; + @InjectMocks + private ViewRetentionTask viewRetentionTask; + + @BeforeEach + void setUp() { + when(stepExecution.getExecutionContext()).thenReturn(executionContext); + } + + @Test + void when_MembersOfFragmentMatchRetentionPoliciesOfView_MembersAreRemovedFromTheView() { + var timeBasedRetentionPolicy = new TimeBasedRetentionPolicy(Duration.ZERO); + + var versionBasedRetentionPolicy = new VersionBasedRetentionPolicy(1); + + var timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); + + when(executionContext.get("retentionPolicy")) + .thenReturn(timeBasedRetentionPolicy) + .thenReturn(versionBasedRetentionPolicy) + .thenReturn(timeAndVersionBasedRetentionPolicy); + when(executionContext.getString("name")) + .thenReturn(VIEW_A.asString()) + .thenReturn(VIEW_B.asString()) + .thenReturn(VIEW_C.asString()); + + for (int i = 0; i < 3; i++) { + viewRetentionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + } + + verify(memberPropertiesRepository).findExpiredMembers(VIEW_A, timeBasedRetentionPolicy); + verify(memberPropertiesRepository).findExpiredMembers(VIEW_B, versionBasedRetentionPolicy); + verify(memberPropertiesRepository).findExpiredMembers(VIEW_C, timeAndVersionBasedRetentionPolicy); + verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_A), anyList()); + verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_B), anyList()); + verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_C), anyList()); + } + + @Test + void given_NoRetentionPolicies_when_RetentionPoliciesExecuted_then_DeleteNoMembers() { + final RepeatStatus returnedRepeatStatus = viewRetentionTask.execute(mock(), new ChunkContext(new StepContext(stepExecution))); + + verifyNoInteractions(pageMemberRepository); + assertThat(returnedRepeatStatus).isEqualTo(RepeatStatus.FINISHED); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciderTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciderTest.java new file mode 100644 index 000000000..6da50ace8 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/decider/RetentionExecutionDeciderTest.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.decider; + +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.services.RetentionPolicyEmptinessChecker; +import be.vlaanderen.informatievlaanderen.ldes.server.maintenance.valueobjects.DecidedFlowExecutionStatus; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.batch.core.job.flow.FlowExecutionStatus; +import org.springframework.batch.core.job.flow.JobExecutionDecider; + +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class RetentionExecutionDeciderTest { + private static final FlowExecutionStatus SKIP_STATUS = DecidedFlowExecutionStatus.SKIP.status(); + private static final FlowExecutionStatus CONTINUE_STATUS = DecidedFlowExecutionStatus.CONTINUE.status(); + private static final RetentionPolicyEmptinessChecker emptyEmptinessChecker = () -> true; + private static final RetentionPolicyEmptinessChecker nonEmptyEmptinessChecker = () -> false; + + @ParameterizedTest + @MethodSource + void testDecide(List emptinessCheckers, FlowExecutionStatus expectedStatus) { + final JobExecutionDecider retentionExecutionDecider = new RetentionExecutionDecider(emptinessCheckers); + + final FlowExecutionStatus result = retentionExecutionDecider.decide(null, null); + + assertThat(result).isEqualTo(expectedStatus); + } + + static Stream testDecide() { + return Stream.of( + Arguments.of(List.of(emptyEmptinessChecker), SKIP_STATUS), + Arguments.of(List.of(nonEmptyEmptinessChecker), CONTINUE_STATUS), + Arguments.of(List.of(emptyEmptinessChecker, nonEmptyEmptinessChecker), CONTINUE_STATUS), + Arguments.of(List.of(nonEmptyEmptinessChecker, nonEmptyEmptinessChecker), CONTINUE_STATUS), + Arguments.of(List.of(emptyEmptinessChecker, emptyEmptinessChecker), SKIP_STATUS) + ); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitionerTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitionerTest.java new file mode 100644 index 000000000..ea0b36743 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/batch/partitioner/RetentionPolicyPartitionerTest.java @@ -0,0 +1,110 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.batch.partitioner; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.EventSourceRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.RetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.ViewRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies.RetentionPolicyCollection; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.item.ExecutionContext; + +import java.time.Duration; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RetentionPolicyPartitionerTest { + private static final String COLLECTION = "collection"; + + @Mock + private RetentionPolicyCollection retentionPolicies; + @InjectMocks + private RetentionPolicyPartitioner retentionPartitioner; + + @Nested + class ViewRetentionPolicy { + private static final ViewName VIEW_A = new ViewName(COLLECTION, "viewA"); + private static final ViewName VIEW_B = new ViewName(COLLECTION, "viewB"); + private static final ViewName VIEW_C = new ViewName(COLLECTION, "viewC"); + + @Test + void given_NonEmptyCollection_testPartitioning() { + RetentionPolicy timeBasedRetentionPolicy = new TimeBasedRetentionPolicy(Duration.ZERO); + RetentionPolicy versionBasedRetentionPolicy = new VersionBasedRetentionPolicy(1); + RetentionPolicy timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); + + when(retentionPolicies.getRetentionPolicies()).thenReturn(Set.of( + new ViewRetentionPolicyProvider(VIEW_A, timeBasedRetentionPolicy), + new ViewRetentionPolicyProvider(VIEW_B, versionBasedRetentionPolicy), + new ViewRetentionPolicyProvider(VIEW_C, timeAndVersionBasedRetentionPolicy) + )); + + final Map result = retentionPartitioner.partition(0); + + assertThat(result) + .hasSize(3) + .containsEntry( + "retention:%s".formatted(VIEW_A.asString()), + new ExecutionContext(Map.of( + "name", VIEW_A.asString(), + "retentionPolicy", timeBasedRetentionPolicy + )) + ); + } + + @Test + void given_EmptyCollection_testPartitioning() { + when(retentionPolicies.getRetentionPolicies()).thenReturn(Set.of()); + + final Map result = retentionPartitioner.partition(0); + + assertThat(result).isEmpty(); + } + } + + @Nested + class EventSourceRetentionPolicy { + + @Test + void given_NonEmptyCollection_testPartitioning() { + RetentionPolicy timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); + + when(retentionPolicies.getRetentionPolicies()).thenReturn(Set.of( + new EventSourceRetentionPolicyProvider(COLLECTION, timeAndVersionBasedRetentionPolicy) + )); + + final Map result = retentionPartitioner.partition(0); + + assertThat(result) + .hasSize(1) + .containsEntry( + "retention:%s".formatted(COLLECTION), + new ExecutionContext(Map.of( + "name", COLLECTION, + "retentionPolicy", timeAndVersionBasedRetentionPolicy + )) + ); + } + + @Test + void given_EmptyCollection_testPartitioning() { + when(retentionPolicies.getRetentionPolicies()).thenReturn(Set.of()); + + final Map result = retentionPartitioner.partition(0); + + assertThat(result).isEmpty(); + } + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProviderTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProviderTest.java new file mode 100644 index 000000000..1ae20e32e --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/EventSourceRetentionPolicyProviderTest.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.time.Duration; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class EventSourceRetentionPolicyProviderTest { + private static final String COLLECTION = "collection"; + private static final RetentionPolicy RETENTION_POLICY = new TimeBasedRetentionPolicy(Duration.ZERO); + private static final EventSourceRetentionPolicyProvider eventSourceRetentionPolicyProvider = new EventSourceRetentionPolicyProvider(COLLECTION, RETENTION_POLICY); + + @Test + void testEquality() { + final EventSourceRetentionPolicyProvider allFieldsEqual = new EventSourceRetentionPolicyProvider(COLLECTION, RETENTION_POLICY); + final EventSourceRetentionPolicyProvider otherRetentionPolicy = new EventSourceRetentionPolicyProvider(COLLECTION, new VersionBasedRetentionPolicy(3)); + + assertThat(eventSourceRetentionPolicyProvider) + .isEqualTo(allFieldsEqual) + .isEqualTo(otherRetentionPolicy) + .isEqualTo(eventSourceRetentionPolicyProvider) + .hasSameHashCodeAs(allFieldsEqual) + .hasSameHashCodeAs(otherRetentionPolicy) + .hasSameHashCodeAs(eventSourceRetentionPolicyProvider); + } + + @ParameterizedTest + @MethodSource + void testInequality(Object other) { + assertThat(other).isNotEqualTo(eventSourceRetentionPolicyProvider); + + if(other != null) { + assertThat(other).doesNotHaveSameHashCodeAs(eventSourceRetentionPolicyProvider); + } + } + + static Stream testInequality() { + return Stream.of( + new EventSourceRetentionPolicyProvider("other", RETENTION_POLICY), + new EventSourceRetentionPolicyProvider("fantasy", new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 2)), + new ViewRetentionPolicyProvider(new ViewName(COLLECTION, "fantasy"), RETENTION_POLICY), + RETENTION_POLICY, + COLLECTION, + null + ); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProviderTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProviderTest.java new file mode 100644 index 000000000..356c4f35f --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/entities/ViewRetentionPolicyProviderTest.java @@ -0,0 +1,59 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.entities; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.time.Duration; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ViewRetentionPolicyProviderTest { + private static final String COLLECTION = "collection"; + private static final String VIEW = "view"; + private static final ViewName VIEW_NAME = new ViewName(COLLECTION, VIEW); + private static final RetentionPolicy RETENTION_POLICY = new TimeBasedRetentionPolicy(Duration.ZERO); + private static final ViewRetentionPolicyProvider viewRetentionPolicyProvider = new ViewRetentionPolicyProvider(VIEW_NAME, RETENTION_POLICY); + + @Test + void testEquality() { + final ViewRetentionPolicyProvider allFieldsEqual = new ViewRetentionPolicyProvider(VIEW_NAME, RETENTION_POLICY); + final ViewRetentionPolicyProvider otherRetentionPolicy = new ViewRetentionPolicyProvider(VIEW_NAME, new VersionBasedRetentionPolicy(3)); + + assertThat(viewRetentionPolicyProvider) + .isEqualTo(allFieldsEqual) + .isEqualTo(otherRetentionPolicy) + .isEqualTo(viewRetentionPolicyProvider) + .hasSameHashCodeAs(allFieldsEqual) + .hasSameHashCodeAs(otherRetentionPolicy) + .hasSameHashCodeAs(viewRetentionPolicyProvider); + } + + @ParameterizedTest + @MethodSource + void testInequality(Object other) { + assertThat(other).isNotEqualTo(viewRetentionPolicyProvider); + + if(other != null) { + assertThat(other).doesNotHaveSameHashCodeAs(viewRetentionPolicyProvider); + } + } + + static Stream testInequality() { + return Stream.of( + new ViewRetentionPolicyProvider(new ViewName(COLLECTION, "fantasy"), RETENTION_POLICY), + new ViewRetentionPolicyProvider(new ViewName("fantasy", VIEW), new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 2)), + new ViewRetentionPolicyProvider(new ViewName("fantasy", "other"), new VersionBasedRetentionPolicy(6)), + new EventSourceRetentionPolicyProvider("other", RETENTION_POLICY), + RETENTION_POLICY, + VIEW_NAME, + VIEW_NAME.asString(), + null + ); + }} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java deleted file mode 100644 index c29c55da0..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/DeletionPolicyCollectionImplTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.DeletionPolicyChangedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DeletionPolicyCollectionImplTest { - private static final String COLLECTION_NAME = "collectionName"; - private List retentionPolicies; - @Mock - private RetentionPolicyFactory retentionPolicyFactory; - private DeletionPolicyCollectionImpl deletionPolicyCollection; - - @BeforeEach - void setUp() { - deletionPolicyCollection = new DeletionPolicyCollectionImpl(retentionPolicyFactory); - } - - @Test - void when_SavePolicies_Then_PoliciesAreSaved() { - retentionPolicies = List.of(ModelFactory.createDefaultModel()); - when(retentionPolicyFactory.extractRetentionPolicy(retentionPolicies)) - .thenReturn(Optional.of(new TimeBasedRetentionPolicy(Duration.ZERO))); - - deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); - - assertThat(deletionPolicyCollection.getEventSourceRetentionPolicyMap()) - .matches(map -> map.containsKey(COLLECTION_NAME)); - } - - @Test - void when_SavePoliciesAreEmpty_Then_PoliciesAreRemoved() { - retentionPolicies = List.of(ModelFactory.createDefaultModel()); - - deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); - - retentionPolicies = List.of(); - - deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); - - assertThat(deletionPolicyCollection.getEventSourceRetentionPolicyMap()) - .matches(map -> !map.containsKey(COLLECTION_NAME)); - } -} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventSourceRetentionPolicyCollectionTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventSourceRetentionPolicyCollectionTest.java new file mode 100644 index 000000000..6b98015e5 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/EventSourceRetentionPolicyCollectionTest.java @@ -0,0 +1,84 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.DeletionPolicyChangedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.EventSourceRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies.EventSourceRetentionPolicyCollection; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EventSourceRetentionPolicyCollectionTest { + private static final String COLLECTION_NAME = "collectionName"; + private List retentionPolicies; + @Mock + private RetentionPolicyFactory retentionPolicyFactory; + private EventSourceRetentionPolicyCollection deletionPolicyCollection; + + @BeforeEach + void setUp() { + deletionPolicyCollection = new EventSourceRetentionPolicyCollection(retentionPolicyFactory); + } + + @Test + void when_SavePolicies_Then_PoliciesAreSaved() { + retentionPolicies = List.of(ModelFactory.createDefaultModel()); + when(retentionPolicyFactory.extractRetentionPolicy(retentionPolicies)) + .thenReturn(Optional.of(new TimeBasedRetentionPolicy(Duration.ZERO))); + + deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); + + assertThat(deletionPolicyCollection.getRetentionPolicies()) + .map(EventSourceRetentionPolicyProvider::collectionName) + .contains(COLLECTION_NAME); + } + + @Test + void when_SavePoliciesAreEmpty_Then_PoliciesAreRemoved() { + retentionPolicies = List.of(ModelFactory.createDefaultModel()); + + deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); + + retentionPolicies = List.of(); + + deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); + + assertThat(deletionPolicyCollection.getRetentionPolicies()) + .map(EventSourceRetentionPolicyProvider::collectionName) + .doesNotContain(COLLECTION_NAME); + } + + @Test + void when_SavePoliciesAreEmpty_Then_PoliciesAreRemovedByCollection() { + retentionPolicies = List.of(ModelFactory.createDefaultModel()); + + deletionPolicyCollection.handleDeletionPolicyChangedEvent(new DeletionPolicyChangedEvent(COLLECTION_NAME, retentionPolicies)); + + retentionPolicies = List.of(); + + deletionPolicyCollection.handleEventStreamDeletedEvent(new EventStreamDeletedEvent(COLLECTION_NAME)); + + assertThat(deletionPolicyCollection.getRetentionPolicies()) + .map(EventSourceRetentionPolicyProvider::collectionName) + .doesNotContain(COLLECTION_NAME); + } + + @Test + void test_IsEmpty() { + assertThat(deletionPolicyCollection.isEmpty()).isTrue(); + } +} \ No newline at end of file diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java deleted file mode 100644 index 3b649d123..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/RetentionPolicyCollectionImplTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; -import org.apache.jena.rdf.model.Model; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class RetentionPolicyCollectionImplTest { - private final RetentionPolicyFactory retentionPolicyFactory = mock(RetentionPolicyFactory.class); - private final RetentionPolicyCollectionImpl retentionPolicyCollection = new RetentionPolicyCollectionImpl( - retentionPolicyFactory); - - @Test - void test_AddingAndDeletingViews() { - ViewSpecification viewSpecification = new ViewSpecification(new ViewName("collection", "additonalView"), - List.of(), List.of(), 100); - when(retentionPolicyFactory.extractRetentionPolicy(viewSpecification)) - .thenReturn(Optional.of(mock(RetentionPolicy.class))); - - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).doesNotContainKey(viewSpecification.getName()); - retentionPolicyCollection.handleViewAddedEvent(new ViewAddedEvent(viewSpecification)); - - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).containsKey(viewSpecification.getName()); - retentionPolicyCollection.handleViewDeletedEvent(new ViewDeletedEvent(viewSpecification.getName())); - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).doesNotContainKey(viewSpecification.getName()); - } - - @Test - void test_InitializingViews() { - List retentionPolicies = new ArrayList<>(); - ViewSpecification viewSpecification = new ViewSpecification(new ViewName("collection", "additonalView"), - retentionPolicies, List.of(), 100); - when(retentionPolicyFactory.extractRetentionPolicy(viewSpecification)) - .thenReturn(Optional.of(mock(RetentionPolicy.class))); - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).doesNotContainKey(viewSpecification.getName()); - - retentionPolicyCollection.handleViewInitializationEvent(new ViewInitializationEvent(viewSpecification)); - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).containsKey(viewSpecification.getName()); - } - - @Test - void test_HandleEventStreamDeletedEvent() { - final String collectionName = "collection"; - when(retentionPolicyFactory.extractRetentionPolicy(any(ViewSpecification.class))).thenReturn(Optional.of(mock(RetentionPolicy.class))); - Stream.of( - new ViewSpecification(new ViewName(collectionName, "view1"), List.of(), List.of(), 100), - new ViewSpecification(new ViewName(collectionName, "view2"), List.of(), List.of(), 100) - ) - .map(ViewInitializationEvent::new) - .forEach(retentionPolicyCollection::handleViewInitializationEvent); - - retentionPolicyCollection.handleEventStreamDeletedEvent(new EventStreamDeletedEvent(collectionName)); - - assertThat(retentionPolicyCollection.getRetentionPolicyMap()).isEmpty(); - } -} diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewRetentionPolicyCollectionTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewRetentionPolicyCollectionTest.java new file mode 100644 index 000000000..c8dad51e8 --- /dev/null +++ b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/ViewRetentionPolicyCollectionTest.java @@ -0,0 +1,84 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories; + +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewAddedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.ViewInitializationEvent; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.ViewRetentionPolicyProvider; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.retentionpolicies.ViewRetentionPolicyCollection; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.creation.RetentionPolicyFactory; +import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.RetentionPolicy; +import org.apache.jena.rdf.model.Model; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ViewRetentionPolicyCollectionTest { + private final RetentionPolicyFactory retentionPolicyFactory = mock(RetentionPolicyFactory.class); + private final ViewRetentionPolicyCollection retentionPolicyCollection = new ViewRetentionPolicyCollection( + retentionPolicyFactory); + + @Test + void test_AddingAndDeletingViews() { + ViewSpecification viewSpecification = new ViewSpecification(new ViewName("collection", "additonalView"), + List.of(), List.of(), 100); + when(retentionPolicyFactory.extractRetentionPolicy(viewSpecification)) + .thenReturn(Optional.of(mock(RetentionPolicy.class))); + + assertThat(retentionPolicyCollection.getRetentionPolicies()) + .map(ViewRetentionPolicyProvider::viewName) + .doesNotContain(viewSpecification.getName()); + retentionPolicyCollection.handleViewAddedEvent(new ViewAddedEvent(viewSpecification)); + + assertThat(retentionPolicyCollection.getRetentionPolicies()) + .map(ViewRetentionPolicyProvider::viewName) + .contains(viewSpecification.getName()); + retentionPolicyCollection.handleViewDeletedEvent(new ViewDeletedEvent(viewSpecification.getName())); + assertThat(retentionPolicyCollection.getRetentionPolicies()) + .map(ViewRetentionPolicyProvider::viewName) + .doesNotContain(viewSpecification.getName()); + } + + @Test + void test_InitializingViews() { + List retentionPolicies = new ArrayList<>(); + ViewSpecification viewSpecification = new ViewSpecification(new ViewName("collection", "additonalView"), + retentionPolicies, List.of(), 100); + when(retentionPolicyFactory.extractRetentionPolicy(viewSpecification)) + .thenReturn(Optional.of(mock(RetentionPolicy.class))); + assertThat(retentionPolicyCollection.getRetentionPolicies()). + map(ViewRetentionPolicyProvider::viewName) + .doesNotContain(viewSpecification.getName()); + + retentionPolicyCollection.handleViewAddedEvent(new ViewInitializationEvent(viewSpecification)); + assertThat(retentionPolicyCollection.getRetentionPolicies()). + map(ViewRetentionPolicyProvider::viewName) + .contains(viewSpecification.getName()); + } + + @Test + void test_HandleEventStreamDeletedEvent() { + final String collectionName = "collection"; + when(retentionPolicyFactory.extractRetentionPolicy(any(ViewSpecification.class))).thenReturn(Optional.of(mock(RetentionPolicy.class))); + Stream.of( + new ViewSpecification(new ViewName(collectionName, "view1"), List.of(), List.of(), 100), + new ViewSpecification(new ViewName(collectionName, "view2"), List.of(), List.of(), 100) + ) + .map(ViewInitializationEvent::new) + .forEach(retentionPolicyCollection::handleViewAddedEvent); + + retentionPolicyCollection.handleEventStreamDeletedEvent(new EventStreamDeletedEvent(collectionName)); + + assertThat(retentionPolicyCollection.getRetentionPolicies()).isEmpty(); + } +} diff --git a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java b/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java deleted file mode 100644 index e6a88a3bd..000000000 --- a/ldes-server-maintenance/ldes-server-retention/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/services/retentionpolicy/execution/RetentionServiceTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.execution; - -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.entities.MemberProperties; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.DeletionPolicyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.MemberPropertiesRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.PageMemberRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.repositories.RetentionPolicyCollection; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timeandversionbased.TimeAndVersionBasedRetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.timebased.TimeBasedRetentionPolicy; -import be.vlaanderen.informatievlaanderen.ldes.server.retention.services.retentionpolicy.definition.versionbased.VersionBasedRetentionPolicy; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.mockito.Mockito.*; - -class RetentionServiceTest { - - public static final String COLLECTION = "collection"; - public static final ViewName VIEW_A = new ViewName(COLLECTION, "viewA"); - public static final ViewName VIEW_B = new ViewName(COLLECTION, "viewB"); - public static final ViewName VIEW_C = new ViewName(COLLECTION, "viewC"); - - private final MemberPropertiesRepository memberPropertiesRepository = mock(MemberPropertiesRepository.class); - private final PageMemberRepository pageMemberRepository = mock(PageMemberRepository.class); - private final MemberRemover memberRemover = mock(MemberRemover.class); - private final RetentionPolicyCollection retentionPolicyCollection = mock(RetentionPolicyCollection.class); - private final DeletionPolicyCollection deletionPolicyCollection = mock(DeletionPolicyCollection.class); - private RetentionService retentionService; - - @BeforeEach - void setUp() { - retentionService = new RetentionService(memberPropertiesRepository, pageMemberRepository, memberRemover, retentionPolicyCollection, deletionPolicyCollection); - } - - @Test - void when_MembersOfFragmentMatchRetentionPoliciesOfView_MembersAreRemovedFromTheView() { - var timeBasedRetentionPolicy = new TimeBasedRetentionPolicy(Duration.ZERO); - - var versionBasedRetentionPolicy = new VersionBasedRetentionPolicy(1); - - var timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); - - when(retentionPolicyCollection.getRetentionPolicyMap()).thenReturn(Map.of( - VIEW_A, timeBasedRetentionPolicy, - VIEW_B, versionBasedRetentionPolicy, - VIEW_C, timeAndVersionBasedRetentionPolicy - )); - - when(deletionPolicyCollection.getEventSourceRetentionPolicyMap()).thenReturn(Map.of()); - - retentionService.executeRetentionPolicies(); - - verify(memberPropertiesRepository).findExpiredMembers(VIEW_A, timeBasedRetentionPolicy); - verify(memberPropertiesRepository).findExpiredMembers(VIEW_B, versionBasedRetentionPolicy); - verify(memberPropertiesRepository).findExpiredMembers(VIEW_C, timeAndVersionBasedRetentionPolicy); - verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_A), anyList()); - verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_B), anyList()); - verify(pageMemberRepository).deleteByViewNameAndMembersIds(eq(VIEW_C), anyList()); - } - - @Test - void when_MembersOfFragmentMatchRetentionPoliciesOfEventSource_MembersAreRemovedFromTheEventSource() { - MemberProperties firstMember = getMemberProperties(1l, 0); - MemberProperties secondMember = new MemberProperties(2L, "coll", "http://ex.com", - LocalDateTime.now().plusDays(1), true, false); - - var timeAndVersionBasedRetentionPolicy = new TimeAndVersionBasedRetentionPolicy(Duration.ZERO, 1); - when(memberPropertiesRepository.retrieveExpiredMembers(COLLECTION, timeAndVersionBasedRetentionPolicy)) - .thenReturn(Stream.of(firstMember, secondMember)); - - when(deletionPolicyCollection.getEventSourceRetentionPolicyMap()).thenReturn(Map.of( - COLLECTION, timeAndVersionBasedRetentionPolicy - )); - - retentionService.executeRetentionPolicies(); - - verify(memberRemover).removeMembersFromEventSource(List.of(firstMember)); - verify(memberRemover).deleteMembers(List.of(secondMember)); - } - - @Test - void given_NoRetentionPolicies_when_RetentionPoliciesExecuted_then_DeleteNoMembers() { - when(retentionPolicyCollection.getRetentionPolicyMap()).thenReturn(Map.of()); - - retentionService.executeRetentionPolicies(); - - verifyNoInteractions(memberRemover); - } - - private MemberProperties getMemberProperties(long memberId, int plusDays) { - return new MemberProperties(memberId, "coll", "http://ex.com", - LocalDateTime.now().plusDays(plusDays), true, true); - } -} \ No newline at end of file diff --git a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java index 032bd5d0e..1cbf73994 100644 --- a/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java +++ b/ldes-server-pagination/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/repositories/PageRepository.java @@ -6,13 +6,12 @@ import java.time.LocalDateTime; import java.util.List; -import java.util.stream.Stream; public interface PageRepository { Page getOpenPage(long bucketId); Page createNextPage(Page parentPage); void markAllPagesImmutableByCollectionName(String collectionName); - Stream getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage); + List getPossibleCompactionCandidates(ViewName viewName, int capacityPerPage); void deleteOutdatedFragments(LocalDateTime deleteTime); void setDeleteTime(List ids, LocalDateTime deleteTime); } diff --git a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/CompactionCandidate.java b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/CompactionCandidate.java index f0c6bbfa4..31064b876 100644 --- a/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/CompactionCandidate.java +++ b/ldes-server-port-fetch/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fetching/entities/CompactionCandidate.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities; -import java.time.LocalDateTime; import java.util.Objects; public class CompactionCandidate { @@ -9,16 +8,11 @@ public class CompactionCandidate { private final String partialUrl; private final long bucketId; private final long nextPageId; - private final boolean isImmutable; - private final LocalDateTime expiration; - public CompactionCandidate(Long id, Integer size, Long nextPageId, Boolean isImmutable, - LocalDateTime expiration, Long bucketId, String partialUrl) { + public CompactionCandidate(Long id, Integer size, Long nextPageId, Long bucketId, String partialUrl) { this.id = id; this.size = size; this.nextPageId = nextPageId; - this.isImmutable = isImmutable; - this.expiration = expiration; this.bucketId = bucketId; this.partialUrl = partialUrl; } @@ -43,10 +37,6 @@ public long getNextPageId() { return nextPageId; } - public boolean isCompactable() { - return isImmutable && expiration == null; - } - @Override public String toString() { return "CompactionCandidate{ id='" + id +"'}"; diff --git a/pom.xml b/pom.xml index a9f1f0da0..c9b3640e5 100644 --- a/pom.xml +++ b/pom.xml @@ -142,6 +142,11 @@ ldes-server-domain ${project.version} + + be.vlaanderen.informatievlaanderen.vsds + ldes-server-maintenance-common + ${project.version} + ${project.groupId} ldes-fragmentisers-impl From 84fbd538fe14145c94b6110859f9c24ef06f9af4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:47:27 +0200 Subject: [PATCH 05/37] chore(deps): bump org.owasp:dependency-check-maven from 9.0.9 to 10.0.4 (#1385) Bumps [org.owasp:dependency-check-maven](https://github.com/jeremylong/DependencyCheck) from 9.0.9 to 10.0.4. - [Release notes](https://github.com/jeremylong/DependencyCheck/releases) - [Changelog](https://github.com/jeremylong/DependencyCheck/blob/main/CHANGELOG.md) - [Commits](https://github.com/jeremylong/DependencyCheck/compare/v9.0.9...v10.0.4) --- updated-dependencies: - dependency-name: org.owasp:dependency-check-maven dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c9b3640e5..3ae570fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 3.25.3 - 9.0.9 + 10.0.4 3.2.3 From 263bc48bcc689a6453149a285226c02c28d7bcc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:50:34 +0200 Subject: [PATCH 06/37] chore(deps): bump com.google.guava:guava from 33.0.0-jre to 33.3.1-jre (#1387) Bumps [com.google.guava:guava](https://github.com/google/guava) from 33.0.0-jre to 33.3.1-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3ae570fe2..5b7fd9088 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 4.10.0 1.1.1 - 33.0.0-jre + 33.3.1-jre 4.13.2 From 80735e8ae13cd77ab4350d330240166a38ea2245 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:51:13 +0200 Subject: [PATCH 07/37] chore(deps): bump org.springframework.data:spring-data-commons (#1388) Bumps [org.springframework.data:spring-data-commons](https://github.com/spring-projects/spring-data-commons) from 3.2.3 to 3.3.4. - [Release notes](https://github.com/spring-projects/spring-data-commons/releases) - [Commits](https://github.com/spring-projects/spring-data-commons/compare/3.2.3...3.3.4) --- updated-dependencies: - dependency-name: org.springframework.data:spring-data-commons dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b7fd9088..165fc907d 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.1.5 - 3.2.3 + 3.3.4 6.0.13 From b7f3c631752152cd5408d93cf748b066a26de20d Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:36:56 +0200 Subject: [PATCH 08/37] fix: flaky test (#1381) --- .../FragmentationServiceTest.java | 73 ++++++++----------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java index 35af7951d..cc6e8374d 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java @@ -2,75 +2,65 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.Step; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.batch.core.repository.JobRepository; import java.util.List; import java.util.Set; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationService.*; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BatchConfiguration.ASYNC_JOB_LAUNCHER; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BucketJobDefinitions.BUCKETISATION_STEP; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.tuple; import static org.mockito.Mockito.*; -@RunWith(SpringRunner.class) -@SpringBatchTest -@EnableAutoConfiguration -@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class}) -@TestPropertySource(properties = {"ldes-server.fragmentation-cron=*/1 * * * * *"}) +@ExtendWith(MockitoExtension.class) class FragmentationServiceTest { - @MockBean(name = BUCKETISATION_STEP) + @Mock Step bucketStep; - @MockBean(name = "paginationStep") + @Mock Step paginationStep; - @MockBean - ServerMetrics serverMetrics; - @MockBean - FragmentationStrategyCollection strategyCollection; - @MockBean + @Mock MemberMetricsRepository memberMetricsRepository; - @MockBean(name = ASYNC_JOB_LAUNCHER) + @Mock JobLauncher jobLauncher; - @MockBean + @Mock JobExplorer jobExplorer; - @Autowired + @Mock + JobRepository jobRepository; private FragmentationService fragmentationService; + @Captor + private ArgumentCaptor captor; + + + @BeforeEach + void setUp() { + fragmentationService = new FragmentationService(jobLauncher, jobRepository, jobExplorer, bucketStep, paginationStep, memberMetricsRepository); + } @Test void when_unprocessedViews_then_triggerJobsForEachView() throws Exception { String collection = "collection"; + List viewNames = List.of(new ViewName(collection, "v1"), new ViewName(collection, "v2")); - when(memberMetricsRepository.getUnprocessedViews()) - .thenReturn(List.of(new ViewName(collection, "v1"), - new ViewName(collection, "v2"))); + when(memberMetricsRepository.getUnprocessedViews()).thenReturn(viewNames); fragmentationService.scheduledJobLauncher(); - ArgumentCaptor captor = ArgumentCaptor.forClass(JobParameters.class); verify(jobLauncher, times(2)).run(any(), captor.capture()); assertThat(captor.getAllValues()) - .extracting(obj -> obj.getString(COLLECTION_NAME), obj -> obj.getString(VIEW_NAME)) - .containsExactlyInAnyOrder( - tuple(collection, "v1"), - tuple(collection, "v2") - ); + .map(params -> new ViewName(params.getString(COLLECTION_NAME), params.getString(VIEW_NAME))) + .containsExactlyInAnyOrderElementsOf(viewNames); } @Test @@ -99,13 +89,10 @@ void when_unprocessedViews_then_triggerJobsForEachViewThatIsntRunningAlready() t fragmentationService.scheduledJobLauncher(); - ArgumentCaptor captor = ArgumentCaptor.forClass(JobParameters.class); - verify(jobLauncher, times(1)).run(any(), captor.capture()); - assertThat(captor.getAllValues()) - .extracting(obj -> obj.getString(COLLECTION_NAME), obj -> obj.getString(VIEW_NAME)) - .containsExactlyInAnyOrder( - tuple(collection, "v2") - ); + verify(jobLauncher).run(any(), captor.capture()); + assertThat(captor.getValue()) + .extracting(params -> new ViewName(params.getString(COLLECTION_NAME), params.getString(VIEW_NAME))) + .isEqualTo(new ViewName(collection, "v2")); } From c2d0e433dfc60fc8f92cb6a8f877fe2a3940194e Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:54:40 +0200 Subject: [PATCH 09/37] chore: db cleanup (#1382) * chore: cleanup old tables and old_id in members table * fix: broken test * fix: invalid sql scripts * chore(deps): update postgres version manually, as dependabot wants to merge to main --- ldes-server-infra-postgres/pom.xml | 2 +- .../postgres/batch/allocations.sql | 3 --- .../batch/delegates/init-writer-test.sql | 9 +++---- .../postgres/MemberPostgresRepository.java | 15 +---------- .../ingest/postgres/entity/MemberEntity.java | 7 +----- .../repository/MemberEntityRepository.java | 2 -- .../server/ingest/MemberRepositorySteps.java | 7 ------ .../postgres/batch/MemberItemReaderTest.java | 6 ++--- .../db/changelog/3_5_0/drop-old-member-id.xml | 13 ++++++++++ .../3_5_0/drop-old-tables-with-cascade.xml | 25 +++++++++++++++++++ .../resources/db/changelog/3_5_0/master.xml | 9 +++++++ .../main/resources/db/changelog/master.xml | 1 + .../pagination/postgres/PaginationSteps.java | 2 +- .../MemberPropertiesPostgresRepository.java | 8 ------ .../RetentionMemberEntityRepository.java | 2 +- .../MemberPropertiesRepository.java | 2 -- .../ingest/repositories/MemberRepository.java | 8 ++---- 17 files changed, 61 insertions(+), 60 deletions(-) delete mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/allocations.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-member-id.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-tables-with-cascade.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/master.xml diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index 2cc0521c7..796ffe5eb 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -22,7 +22,7 @@ - 42.6.0 + 42.6.1 3.7.4 diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/allocations.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/allocations.sql deleted file mode 100644 index 960afe6bc..000000000 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/allocations.sql +++ /dev/null @@ -1,3 +0,0 @@ -insert into fetch_allocation (id, collection_name, fragment_id, member_id, view_name) -values ('1', 'es', 'v1/x', 1, 'es/v1'), - ('2', 'es', 'v1/x', 2, 'es/v1'); \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql index 56591c9cb..e341af42d 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/resources/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/batch/delegates/init-writer-test.sql @@ -1,25 +1,22 @@ -INSERT INTO members (member_id, subject, old_id, collection_id, is_in_event_source, member_model, timestamp, +INSERT INTO members (member_id, subject, collection_id, is_in_event_source, member_model, timestamp, transaction_id, version_of) VALUES (1, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600000', - 'mobility-hindrances/https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600000', 1, true, E'\\x7212700A2412220A20383962633766336534653666643530393261313039333638386438313639376512310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A151A130A0A48696E6465725A6F6E6512056E6C2D6265B70112B4010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303030122F0A2D0A2B687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F636F6E7472696275746F721A2412220A206464353034373238373065356361626163393133343534376263316239313031B60112B3010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303030122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D732376657273696F6E4E6F7465731A281A260A1D4D6F62696C69747948696E6472616E63655A6F6E65576173416464656412056E6C2D6265D40112D1010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012250A230A21687474703A2F2F7075726C2E6F72672F64632F7465726D732F6D6F6469666965641A4B1A490A1C323032322D30352D32305430393A35383A31352E383634363433335A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65DA0112D7010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303030122D0A2B0A29687474703A2F2F7777772E77332E6F72672F6E732F70726F762367656E657261746564417454696D651A491A470A1A323032342D30362D32365431323A34373A30322E3637313338301A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65C20112BF010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303030123A0A380A3668747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423496E6E616D652E7374617475731A2412220A206532656662383639303231313564646462386230666262313632376435623932D50112D2010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A3E0A3C0A3A68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974234D6F62696C69746569747368696E646572B90112B6010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974237A6F6E651A2412220A203630363733626261353364353661333739376137613730383633393961343832D30112D0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012240A220A20687474703A2F2F7075726C2E6F72672F64632F7465726D732F637265617465641A4B1A490A1C323032322D30352D32335430393A35383A31352E383631303839365A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65E00112DD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F697356657273696F6E4F661A540A520A5068747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F3130383130343030BE0112BB010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236265686565726465721A2412220A206464353034373238373065356361626163393133343534376263316239313031B00112AD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73236964656E7469666965721A2412220A203533653066323335336238623533376162393162303364623032393934393239CA0112C7010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012300A2E0A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449641A361A340A0831303831303430301A28687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D6123696E7465676572BC0112B9010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012340A320A3068747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423706572696F64651A2412220A2034326535316265343464656561363762303066643533383136333935386565629101128E010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012260A240A22687474703A2F2F7075726C2E6F72672F64632F7465726D732F737562737472696E671A071A050A012648019E01129B010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303012280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F6465736372697074696F6E1A121A100A0C6F6D73636872696A76696E674801B30112B0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303030122B0A290A27687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F63726561746F721A2412220A20646435303437323837306535636162616339313334353437626331623931303187011284010A2412220A20363036373362626135336435366133373937613761373038363339396134383212360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E652E747970651A2412220A2038396263376633653465366664353039326131303933363838643831363937659101128E010A2412220A20363036373362626135336435366133373937613761373038363339396134383212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E657612740A2412220A20363036373362626135336435366133373937613761373038363339396134383212260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2367656F6D657472791A2412220A20373630343563303738343364616636646461623338626130663632666561363184011281010A2412220A20363036373362626135336435366133373937613761373038363339396134383212330A310A2F68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236765766F6C671A2412220A2036313063336161323561343134326534383339643036393337303133343137378B011288010A2412220A20363130633361613235613431343265343833396430363933373031333431373712310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A2D1A2B0A2242657065726B746520646F6F7267616E6720766F6F7220766F657467616E6765727312056E6C2D626571126F0A2412220A20653265666238363930323131356464646238623066626231363237643562393212310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A141A120A09496E206F706D61616B12056E6C2D626580960112FC95010A2412220A203736303435633037383433646166366464616233386261306636326665613631122E0A2C0A2A687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C236173574B541AA295011A9E95010AE994013C687474703A2F2F7777772E6F70656E6769732E6E65742F6465662F6372732F455053472F392E392E312F33313337303E20504F4C59474F4E2028283132313337352E38333834393431373631203135373732352E30323930363930313833382C203133313337352E33323039313439333632203136373732342E32343835373332383630382C203132313336392E3935323931343933353438203135373731372E363031353733323836322C203132313336392E3531353937363937333239203135373731372E31323637393334323037362C203132313336332E3034363937363936393035203135373731302E39343937393334313936382C203132313336322E38303132303432303131203135373731302E37333031333734393031342C203132313335352E38333932303432303336203135373730342E39303931333734393135342C203132313335352E3831353036393533353838203135373730342E383839303836353738312C203132313335352E3636373837333938333538203135373730342E373637353736313533322C203132313334362E35303336373730343931203135373639372E313838303637343330332C203132313334352E37343035373034373534203135373639362E36363736333637343632382C203132313333342E3738333537303437333134203135373639302E35393536333637343536352C203132313333342E3234343534383433323137203135373639302E33333737343939333131382C203132313332332E3337373534383433333438203135373638352E39313537343939323930362C203132313332332E3132373937333232303437203135373638352E38323138363930343033342C203132313236332E3230353937333232323038203135373636352E30383738363930343239372C203132313236332E31393133363632313639203135373636352E30383238343030333231382C203132313236312E3435373336363231323037203135373636342E34383838343030333136372C203132313236312E3434373933313236373834203135373636342E34383536313834393836342C203132313232312E363835323134333534203135373635302E393533303038383733372C203132313231392E3739333233333631353134203135373635302E323731313532333233312C203132313231392E3639363936343433363436203135373635302E323337353632363433332C203132313230352E3531393635363237303137203135373634352E34353234383337333134372C203132313138362E3039353334333437363533203135373633382E38343434313834393433382C203132313136312E3232323730393539333933203135373633302E333930343238303535312C203132313134322E3833313030323232303739203135373632322E3933303031313039312C203132313132372E3034303939303235393337203135373631342E35393537363337343833372C203132313131392E34363930323831303638203135373631302E353835373432363038392C203132313039352E3835353130383739313139203135373539362E37323035343333353332342C203132313037382E3335323535353639343132203135373538352E35343534323635393835382C203132313036322E38343332323738343937203135373537332E38383836373831333639352C203132313034392E3039313536343136303637203135373536312E37323730373531353738332C203132313032352E3336393033373434383237203135373533382E39383635343833393835332C203132313032342E32343138323730353833203135373533372E393036333436363935312C203132313032322E38333834323735303239203135373533362E35363039323233333730382C203132313031352E3730343033373836373431203135373532392E373139353037363334372C203132303938382E3630323431373431333039203135373530332E37313938373137343537362C203132303935352E3739363337393738313233203135373437322E323635383736353839382C203132303935352E3737303232303035323538203135373437322E32343039373536393438352C203132303931352E3431313637393232323933203135373433342E31303134303936323332382C203132303835342E3439383539333438303237203135373337362E35303832373430343735322C203132303834312E3931333034383636343333203135373336342E36303237303436343230322C203132303834312E38383639393733353037203135373336342E353738323336363638392C203132303833372E3134383136313533353832203135373336302E31353931383733323836322C203132303831302E3236393636323232343639203135373333352E30303733393838383839332C203132303830352E34333530303239333033203135373333302E34373636353533303935332C203132303735342E3132303239363032363936203135373238322E34353931323034353034352C203132303733362E3031373036373835323732203135373236352E35323334353637393831372C203132303733322E3633303839323933313135203135373236322E313035393031313539372C203132303732332E3536333835343038313434203135373235322E393533383939343330392C203132303732322E3034323333303639393931203135373235312E34313133343533383237342C203132303731302E3732303035323035383538203135373233372E39353836383830343931342C203132303730312E36353938373539303734203135373232352E313237333330333331312C203132303639342E35323531373534373331203135373231322E383132323830363638322C203132303639342E31353030353031353738203135373231322E303839393436333338322C203132303636362E3938373738333437333634203135373135392E35393332393633333532352C203132303632382E3830303239373638393537203135373038352E38323633353737303031332C203132303631342E3730363533303136303639203135373035382E363038383036353730342C203132303630382E3634353031383839353834203135373034362E38393037353133383630332C203132303630382E3633353532383136313835203135373034362E38373234393631323836342C203132303539372E3231333634383831373735203135373032352E30313237323730353737372C203132303538332E3135383937323836333934203135363939382E31303733343733383734342C203132303537372E3038393137363230313233203135363938362E34383237333638313037352C203132303534362E3136363735313434383937203135363932372E323934393233353130382C203132303532392E3839323836333630333737203135363839362E313338313338323239362C203132303532362E3437383130383232353532203135363838392E363035363932363138322C203132303532362E3436363638363234333235203135363838392E35383339373233303435362C203132303531352E3334393638363234343536203135363836382E353638393732333037372C203132303530392E3535323431353732333034203135363835372E36313534363039353531362C203132303438372E3634363634383236303237203135363831362E32303839303035303435342C203132303438332E3530323730313031303935203135363830382E33373534353138393039382C203132303437392E3532343432393832353036203135363830302E3737363339343734312C203132303436382E3639313137373038363232203135363738302E30353038323433353530372C203132303436372E38343435333831353436203135363737382E343332363032333335322C203132303436372E3036383739323836373639203135363737362E39343930383934343130372C203132303434352E3634393539303634313631203135363733362E30313137303236323331362C203132303433342E35363631363337303435203135363731342E383132363234303637342C203132303433332E3631343432353638343333203135363731322E39383337333836303933332C203132303433302E3739343234303839333939203135363730362E383832383134363633392C203132303432382E35393537313533393733203135363730302E34393331313730343138362C203132303432362E3939303037353734383533203135363639332E3737363136303030352C203132303432362E3031373933343736323631203135363638362E37383737383536313336352C203132303432352E3732343336393430323135203135363637392E35373731343530383838372C203132303432352E39383436393835333539203135363637342E3634353139353338322C203132303432362E3132303535393934393738203135363637322E31393531333038393030332C203132303432372E3234393735363234393036203135363636342E35393938353033353338372C203132303432382E3330353136323637373136203135363635392E323134313434363131312C203132303433302E3035353238373033373636203135363635302E33333334313239363933342C203132303433312E3538373535343435333336203135363634342E38373936363136353433332C203132303433342E3737343438363238363535203135363633332E35343039303432303934382C203132303433342E3832313536313831313036203135363633332E333630353737383931352C203132303433382E3336383536313830393435203135363631382E36353735373738383938382C203132303433382E3434363534343839313335203135363631382E323636353230353438372C203132303433382E3439333534343838393734203135363631372E393639353230353530332C203132303433382E3439373636373631313732203135363631372E39343330303738363635362C203132303433382E3538323034373330343535203135363631372E333930363135323434312C203132303433392E3430303338313438323136203135363631322E31383332313535363630342C203132303434302E3139373930323931313233203135363630372E30383538393031383736382C203132303434302E3230303038333231333332203135363630372E30373138323337373837342C203132303434302E3335363038333230393039203135363630362E303535383233373736312C203132303434302E34303334373632323138203135363630352E363231323332393433382C203132303434312E3036373437363232363334203135363539352E34303332333239343533332C203132303434312E3037353036393031373734203135363539342E39303738323339323637372C203132303434302E3737393036393031333932203135363538362E32363638323339323738372C203132303434302E3733323339333930373635203135363538352E373335343333313830352C203132303433392E3639323339333930383534203135363537382E343037343333313738392C203132303433392E3537373434333336303631203135363537372E38333738303231303036362C203132303433372E3934353434333336313333203135363537312E36333438303231303237382C203132303433372E3835373533303938343933203135363537312E333338323538363133382C203132303433312E303231353330393832203135363535302E363530323538363132382C203132303433302E3937313537323632343731203135363535302E353036343534373031312C203132303432322E3634333631313130373636203135363532372E36363235363032343436372C203132303431382E3331333634393434393138203135363531352E37383336363534343633362C203132303431382E33303439343131313138203135363531352E37353939363130353432352C203132303431342E36393735363337343835203135363530362E30313636343237343635322C203132303431322E33393531383531383033203135363439392E37383433323439313337352C203132303431302E3930313136353632373233203135363439352E37343239313231333637322C203132303430362E3130353331313430333835203135363438322E33373436313832353030362C203132303339372E3930383838353934343832203135363435392E35343534333237353032352C203132303339372E38383839333430343737203135363435392E34393038363036333639362C203132303338352E3332363933343034383732203135363432352E37343038363036333639362C203132303338332E3933353435303831373336203135363432322E30303535363239303734332C203132303338332E3837363534353738313832203135363432312E383534393331393237362C203132303337362E32373535343537373833203135363430332E33303739333139323932322C203132303337362E3138333634343733333734203135363430332E30393735383139323438372C203132303337302E3635323634343733373937203135363339312E31393035383139323333352C203132303337302E3236333938353438393239203135363339302E353032323138333733392C203132303336342E3333333938353438393539203135363338312E37303532313833373535312C203132303336342E30343432323635363538203135363338312E33313733303938333538342C203132303335372E3134353232363536313837203135363337322E39353833303938333437342C203132303335362E3833333138333733393338203135363337322E363134313337313433352C203132303335322E3535373138333733383834203135363336382E33313731333731343531322C203132303335302E3330313334373231373336203135363336362E303531323936373039332C203132303334392E3834393835303336383733203135363336352E363439353634373539392C203132303334322E30303538353033363435203135363335392E343737353634373631352C203132303334312E3437313135353237313635203135363335392E31313033373834333331362C203132303333332E31383235303031323831203135363335342E313737333434393133322C203132303333322E3732323832353632323834203135363335332E39303133343337363937342C203132303333322E3530333137383530383132203135363335332E373736383935343330312C203132303332332E3830373137383531333235203135363334392E313335383935343237352C203132303332332E3433343937383936343631203135363334382E393536363031333839332C203132303331392E3439393937383936323232203135363334372E32353736303133393230382C203132303331332E3337323037343837363832203135363334342E36313332313131343134362C203132303331322E3834363433353237383734203135363334342E343230353137313531392C203132303330342E3136363433353237393034203135363334312E37373935313731353330322C203132303330332E3731323636353135353835203135363334312E36363433363037303831382C203132303330312E3334353636353134393731203135363334312E31383033363037303730382C203132303330312E3332373730303038323339203135363334312E313736373231353335362C203132303239332E3737323730303038323639203135363333392E363630373231353336372C203132303239332E3337383237393735353938203135363333392E35393738343634393232322C203132303238342E363931323739373537203135363333382E35363638343634393237332C203132303238342E3533323439343231313938203135363333382E353530353637303031372C203132303238312E3435343439343231303337203135363333382E32383435363639393930362C203132303238312E3434323332343230323034203135363333382E32383335333032343339332C203132303236382E3739333332343230353536203135363333372E32323135333032343439352C203132303236382E3338393533383431383632203135363333372E323034303231313336352C203132303235322E3232353533383432313534203135363333372E31353730323131333831322C203132303235322E3039383732323639363932203135363333372E31353832363037373938332C203132303233312E323632373232363934203135363333372E363236323630373738332C203132303233312E3139303738313732363638203135363333372E36323833393437383936372C203132303231332E3337303738313732363338203135363333382E323835333934373931322C203132303231332E3230323238343134383331203135363333382E323934343536333634382C203132303139382E3339373238343134383631203135363333392E33343134353633363331382C203132303139382E3230313830333130333833203135363333392E333539313432383432372C203132303138382E3134373830333130363136203135363334302E343638313432383433382C203132303138382E3131373538303430353531203135363334302E343731353639363135392C203132303138362E33373435383034303438203135363334302E363734353639363133382C203132303138362E3231333434353636393734203135363334302E36393539393635323136322C203132303137312E3639303434353636373833203135363334322E38363739393635323337342C203132303137312E3532393339313734303237203135363334322E38393437373832393336322C203132303136362E3036383339313733373335203135363334332E38393437373832393336322C203132303134312E3433313437333338343935203135363334382E34323634393734323332342C203132303134312E3336343630363237373737203135363334382E34333932363831383030322C203132303131302E3034343630363237373437203135363335342E363432323638313737392C203132303130332E3633393034333239393036203135363335352E39313335373739373235362C203132303039392E3634353434393636303132203135363335362E37303534393733393130392C203132303039392E3432353333313533363538203135363335362E37353433323838383837382C203132303038312E3438373333313533353537203135363336312E31363033323838383832382C203132303038312E3334333436323035393837203135363336312E3139373934333731372C203132303036382E3738303436323035383836203135363336342E363832393433373136342C203132303036382E3639363838353337303938203135363336342E37303639313235353138362C203132303035342E3732303838353337343932203135363336382E38343639313235353234362C203132303035342E3135323738383635363838203135363336392E30353332393733343435372C203132303034382E3233383738383635323335203135363337312E36313632393733343535382C203132303034382E3233333233373037303234203135363337312E36313837303732373334322C203132303033382E3130303233373036383932203135363337362E303234373037323732392C203132303033372E39393838393834373832203135363337362E30373031313536383037352C203132303032322E3836303330373731373338203135363338332E303536333931363030312C203132303030332E3834373533343237303535203135363339312E313834353036373231322C203132303030332E3733333439333934313032203135363339312E32333439353130303833342C203131393939392E33323734393339333738203135363339332E32343939353130303532312C203131393939392E33313532383334303236203135363339332E32353535353531303837372C203131393939362E35373534323130343839203135363339342E35313734393137313535322C203131393939352E3735373031353330353533203135363339342E38393433373630313036332C203131393939342E32383130313431383535203135363339352E35373035383232363530362C203131393939332E3639383235353934313532203135363339352E37383032353538343635362C203131393939332E3537363436393234323736203135363339352E38323538373135323730382C203131393939312E3136343538383135383038203135363339362E37363531393537383938332C203131393938382E3733303333323537303134203135363339372E35353938363132323831362C203131393938382E3539373133373931333034203135363339372E36303534323734303430382C203131393938352E34373336373837333237203135363339382E37323333323739343336352C203131393937392E313336353232393139203135363430302E373937323730393730332C203131393937342E3038383633323838373432203135363430322E31333334343835333230382C203131393937322E3739343635393438343835203135363430322E34373539323333383935372C203131393937302E3034303436393739313232203135363430322E383430313538323437362C203131393937302E3033323934323731373638203135363430322E38343131353935303038322C203131393936362E3533333333323631373838203135363430332E33303933383331393939362C203131393936302E35313530343736313534203135363430332E383036343832393230352C203131393935352E3638323236323638343034203135363430342E30323931323634313635342C203131393934392E3932373739313837363434203135363430332E38383534393438323232332C203131393933362E3334333130323734313234203135363430312E38303430373838323334322C203131393933332E3136373530343937343534203135363430312E30373933333736383533342C203131393933332E3136313636393935323032203135363430312E30373830303936373832382C203131393932352E3339383931353130343236203135363339392E333136313530363536372C203131393931312E3432333939343233373738203135363339362E30333431393933363834342C203131393839372E3039383236363230343231203135363339322E333833373038303234342C203131393838372E3536303836303530343637203135363338392E35333937303138343934362C203131393838322E3834383433363831383431203135363338382E303032363738323934372C203131393837392E3438393430333536383132203135363338362E38383230303532373731382C203131393837392E3436333636313533363331203135363338362E38373334393434363434342C203131393837352E3931393631313339393936203135363338352E373132343132363737352C203131393837302E3037393131393930323437203135363338332E34343431353630393135342C203131393836392E3937383436373038323536203135363338332E343036333036333230352C203131393836302E3138333935313837323734203135363337392E38343238393030383834352C203131393834382E3131363031313238313633203135363337342E39313233373334343336332C203131393832332E3438393836343837343931203135363336342E35373735343437313832322C203131393832332E33383936323335383839203135363336342E353336373436383035382C203131393831372E31373039353132393431203135363336322E30383338373630363137382C203131393831362E3032333635373736383337203135363336312E363331313534383338392C203131393739362E3736333033363038313438203135363335342E30323733303431383939382C203131393739322E3932313639323739393935203135363335322E35313130363030333636332C203131393738342E3636373431343032393537203135363334382E353235333137363336332C203131393737392E3232323837353736313638203135363334332E39323636313635313436382C203131393737382E37313931363636323736203135363334332E32383330353633323735382C203131393737382E3034323239323836363738203135363334322E35373431323930343531382C203131393737372E3234303132303139333636203135363334322E303130383735313231372C203131393737362E33343334373536353432203135363334312E36313439343030383935322C203131393737352E3338363831363739353233203135363334312E34303135333935303938382C203131393737342E3430363930373438303138203135363334312E33373838373432343734332C203131393737332E34343134303530373435203135363334312E35343738313533313534352C203131393737322E3532373431333239343137203135363334312E39303138373034303334342C203131393737312E3730303035363333303735203135363334322E34323734333333373239342C203131393737302E3939313132393034383334203135363334332E313034333037313333382C203131393737302E3432373837353132343835203135363334332E393036343739383036392C203131393737302E3033313934303039323637203135363334342E38303331323433343633362C203131393736392E3831383533393531333036203135363334352E37353937383332303533342C203131393736392E3739353837343235303539203135363334362E37333936393235323033382C203131393736392E39363438313533313836203135363334372E37303531393439323630362C203131393737302E33313838373034303636203135363334382E36313931383637303633382C203131393737302E3834343433333337363131203135363334392E34343635343336363938322C203131393737312E3636353633333337343337203135363335302E34393537343336373139332C203131393737322E3337363634393131323331203135363335312E323333373732323338362C203131393737382E3639373634393131343633203135363335362E35373237373232333634322C203131393737392E3734393834383530313531203135363335372E32353535363230373439352C203131393738382E3733393834383439363134203135363336312E35393635363230373638332C203131393738392E3037383234363438363734203135363336312E37343438303733353530382C203131393739332E3039313130353135343038203135363336332E33323837353135363539352C203131393831322E33353139363339323036203135363337302E39333236393538313039322C203131393831332E3530303732303830373437203135363337312E33383539393435343632332C203131393831392E3637303033343938363234203135363337332E38313933393637303538332C203131393834342E32363831333531323634203135363338342E313432343535323834332C203131393834342E3331313932383237323737203135363338342E31363035393032353330342C203131393835362E3439313932383237323437203135363338392E31333638393032353034362C203131393835362E3637333533323932203135363338392E32303639393336373838382C203131393836362E3530393030323939313134203135363339322E3738353331303030372C203131393837322E3432343838303039383139203135363339352E303832383433393037332C203131393837322E3637383333383436303839203135363339352E31373335303535333339352C203131393837362E3333373435353930333531203135363339362E33373232383530323530352C203131393837392E3639393539363432373437203135363339372E34393339393437323131322C203131393837392E3733313536353432393338203135363339372E35303435343130363235322C203131393838342E3532303536353433333931203135363339392E30363635343130363532332C203131393838342E3634323139373334323031203135363339392E31303435303532393234362C203131393839342E33333731393733343233203135363430312E39393535303532393133362C203131393839342E3533313335313430343334203135363430322E30343931363639353636352C203131393930392E3030303335313430313132203135363430352E37333631363639353536342C203131393930392E3039313837313233353735203135363430352E37353835373139343238362C203131393932332E3133303837313233323833203135363430392E30353535373139343439382C203131393932332E3136373333303034343136203135363430392E30363339393033323236342C203131393933302E3934353431323132353531203135363431302E383239333238303630392C203131393933342E3239343439353032323936203135363431312E35393336363233313530382C203131393933342E36343937343830303936203135363431312E36363133323432393434332C203131393934382E3732373734383031313231203135363431332E38313833323432393232322C203131393934392E3336303233383835333937203135363431332E383734343433323231362C203131393935352E3631303233383835333937203135363431342E30333034343332323438322C203131393935352E39363531303239333835203135363431342E30323637303234353838352C203131393936312E3036363130323934323032203135363431332E37393137303234353537322C203131393936312E3234373538393533373537203135363431332E37383030333036303738362C203131393936372E3438323538393533363937203135363431332E3236353033303631312C203131393936372E3733343035373238363937203135363431332E323337383430343939312C203131393937312E33353532393431323632203135363431322E37353333343339383039332C203131393937342E3432313533303231333733203135363431322E333437383431373439372C203131393937352E30343532393639343031203135363431322E32323435373030343033362C203131393937362E3634373239363933383139203135363431312E38303035373030343136342C203131393938312E3833343433373138383337203135363431302E34323735333239313936322C203131393938322E3131303137313433333335203135363431302E33343539393334353731362C203131393938382E3634393137313433303433203135363430382E32303539393334353635362C203131393938382E3737383836323038313534203135363430382E31363135373235393638342C203131393939312E3930303538353531303634203135363430372E303434323933323930372C203131393939342E3430303636373432373132203135363430362E32323831333837373433342C203131393939342E3636333533303735393332203135363430362E31333431323834373338322C203131393939372E3134343933343035323337203135363430352E31363737323832393139372C203131393939372E38363437343430353935203135363430342E39303837343431353237352C203131393939382E3235343532343631313432203135363430342E37343936363733303336342C203131393939392E3932363532343630393831203135363430332E39383336363733303437342C203131393939392E3933353434303736303431203135363430332E39373935373139323430352C203132303030302E3735383434303736363739203135363430332E363030353731393233342C203132303030332E3439323631353039363331203135363430322E333431323535313339342C203132303030372E3833353830303433373334203135363430302E33353439383232323932342C203132303032362E3835363436353733333039203135363339322E323233343933323738362C203132303032362E3938363130313532313231203135363339322E31363538383433313639362C203132303034322E3133383638343335383137203135363338352E31373331353132323432372C203132303035322E3231373938373838313834203135363338302E37393034393933363235372C203132303035372E3835313431313631313437203135363337382E33343930393530363533382C203132303037312E3439353433303438353139203135363337342E33303734333532303833382C203132303038332E3934343838323930393638203135363337302E383533393333353231382C203132303130312E37303131363030343335203135363336362E343932353638393836332C203132303130352E3538343535303334343035203135363336352E373232353032363038332C203132303131312E3938383337383434303032203135363336342E34353135333637393432382C203132303134332E3237343030333634343632203135363335382E32353533343438303539322C203132303136372E38373135363735383139203135363335332E37333038363239323332362C203132303137332E3235303239393935313534203135363335322E373435393237343936332C203132303138372E3631323136313133393439203135363335302E353938303236373530382C203132303138392E3235393331333436303737203135363335302E34303631383937333535342C203132303139392E32303036323939383137203135363334392E33303936313932313238342C203132303231332E3832333534333532353936203135363334382E32373534393632353034382C203132303233312E3532333235353437303931203135363334372E36323239333131313136332C203132303235322E3235393837373330373131203135363334372E31353731363332353835352C203132303236382E3135383231343536393932203135363334372E32303333393037393034372C203132303238302E3539393539303135363032203135363334382E323437393538383038382C203132303238332E3539313938333036393936203135363334382E35303635363036363739372C203132303239322E3030313235333435363238203135363334392E35303435393838353839342C203132303239392E33353133313430343833203135363335302E39373934373533343436382C203132303330312E3437393735313238303935203135363335312E34313436393434343931372C203132303330392E3636373334323530323931203135363335332E393035383732383337332C203132303331352E3533363437333035343332203135363335362E34333835393337373633372C203132303331392E32383132313434383536203135363335382E303535343436353533342C203132303332372E3638333432363536323336203135363336322E353339363533383038382C203132303332382E3034333137343337323333203135363336322E37353536353632333137382C203132303332382E3035393834343732343132203135363336322E37363536323135363838362C203132303333362E3037373536333538393132203135363336372E353337343036303431372C203132303334332E34323832333936363037203135363337332E33323132333733333636342C203132303334352E3436393233343530323734203135363337352E33373132383331353834392C203132303334392E3538303739313538313637203135363337392E35303330333236333434372C203132303335362E3137373439323030393334203135363338372E34393537353839343939382C203132303336312E3735303339313638363531203135363339352E37363330303939353339352C203132303336372E3036353938393831343734203135363430372E323036323938313432322C203132303337342E3539323738333339363032203135363432352E353732323239303033332C203132303337352E3935353330373438383638203135363432392E32323937383832383030332C203132303338382E3530363933313435383637203135363436322E393531393131323731372C203132303339362E3639323930313233303731203135363438352E37353139373434383337352C203132303430312E3439363638383539343234203135363439392E31343233383137353038362C203132303430312E3531333139383836393734203135363439392E31383737313433313939372C203132303430332E3031353030363739353432203135363530332E32353031393437353736322C203132303430352E3331373831343831393232203135363530392E34383336373530383534372C203132303430382E3932323637343738323038203135363531392E32323031393737343435352C203132303431332E3234383335303534393831203135363533312E30383733333435353239382C203132303432312E3535303335333436353436203135363535332E38363030323336353737362C203132303432382E3331333932343538323135203135363537342E33323838323938363135372C203132303432392E3833323530323332333239203135363538302E31303037323739373939372C203132303433302E3739343037353136303635203135363538362E38373631313831333335342C203132303433312E3037323433393334333134203135363539352E30303232383332333334352C203132303433302E3433383731313734323139203135363630342E37353434333437313135362C203132303433302E3331363939363932383837203135363630352E35343731343134363739382C203132303432392E3532313335373234393832203135363631302E36333234343730333039322C203132303432382E3730313631383532333338203135363631352E38343837383434333534382C203132303432382E36393833333233393036203135363631352E38363939393231333434352C203132303432382E3631343335383138303031203135363631362E343139373330323632392C203132303432382E3630303634333630313937203135363631362E35303633393437323636332C203132303432352E3132323239353239363538203135363633302E393234383230323032352C203132303432312E3936303531333731323835203135363634322E31373430393537393033372C203132303432302E3337343337373337383538203135363634372E38313935383039363234342C203132303432302E3238323335313830383837203135363634382E32303532343436393538362C203132303431382E3439333335313830343334203135363635372E32383332343436393734362C203132303431382E3439323332353630323038203135363635372E32383834363636343738352C203132303431372E3431343332353630373933203135363636322E37383934363636343631352C203132303431372E3337353335383034303932203135363636332E30313537323735333838322C203132303431362E3139353335383034313232203135363637302E39353237323735343135332C203132303431362E3134383636393637393536203135363637312E34313131363434303831342C203132303431352E3939393636393637353633203135363637342E30393831363434303731332C203132303431352E3939383935303931323733203135363637342E3131313434353738332C203132303431352E3732353935303931303833203135363637392E323833343435373831342C203132303431352E3732333133383638323536203135363637392E37353033393534393733372C203132303431362E3033363133383638333537203135363638372E34333833393534393833382C203132303431362E30373936383638313633203135363638372E39323339303739343132372C203132303431372E3131383638363832303833203135363639352E33393239303739343137372C203132303431372E3230383030393037303433203135363639352E38363634363235353638352C203132303431382E3932363030393036353138203135363730332E30353334363235353538342C203132303431392E3036313033373337323336203135363730332E35313737363635353232382C203132303432312E3432313033373337313737203135363731302E33373637363635353333382C203132303432312E3631303434313835323734203135363731302E38343739373238323533342C203132303432342E3538363434313835363236203135363731372E32383539373238323633362C203132303432342E3638393632383532393833203135363731372E34393631333334323930332C203132303432352E3639373632383533313134203135363731392E3433333133333432382C203132303432352E3730323035323137353838203135363731392E343431363134323033362C203132303433362E3738383035323137313335203135363734302E363435363134323033352C203132303435382E3230373438363835393839203135363738312E353833343435353132392C203132303435382E3938333230373132353933203135363738332E30363639313035353738332C203132303435392E3832393236393632353238203135363738342E363834303330323733362C203132303437302E3636323832323930383437203135363830352E34313031373536343431322C203132303437342E3634383331383335343034203135363831332E30323330333433383937342C203132303437342E36353833323034353036203135363831332E30343230343033343733382C203132303437382E3830373332303434373039203135363832302E38383530343033343538362C203132303530302E3731333338323937353738203135363836322E32393231353835333736362C203132303530362E3531303534393633383833203135363837332E323435343733353938322C203132303531372E3632313537363131313439203135363839342E323439313831353531342C203132303532312E3032393531333937313233203135363930302E37363835383436333936382C203132303533372E3330333133363339383535203135363933312E393234383631373731342C203132303536382E3232353039323232383233203135363939312E313131373737323738392C203132303537342E3239343832333739343635203135373030322E37333632363331383630332C203132303538382E3335303233303533333935203135373032392E36343330343139393234342C203132303539392E3736373730373639383839203135373035312E34393433383630303933322C203132303630352E3832343938313131303432203135373036332E323034323438363136352C203132303631392E3931393833303633353139203135373039302E343233383930313837372C203132303635382E3130363435393337303438203135373136342E31383931373330313438332C203132303638352E3237303231363532373637203135373231362E36383837303336363235342C203132303638352E3237333638383232313332203135373231362E36393534303130333838332C203132303638352E3730333638383232313032203135373231372E35323334303130343034342C203132303638352E38313436323034393831203135373231372E37323534373936373938322C203132303639332E31313936323034393738203135373233302E33333434373936383039332C203132303639332E3336313537323731373937203135373233302E37313139393936313330342C203132303730322E3637333537323731363936203135373234332E38393939393936313033322C203132303730322E3933323536343939323732203135373234342E32333536333436303437332C203132303731342E3531383536343938383139203135373235382E30303136333436303733352C203132303731342E3738343238333337323439203135373235382E32393331383436333834362C203132303731362E3434383238333337373032203135373235392E39383031383436333734352C203132303731362E3435363031373036393732203135373235392E39383830303739393333352C203132303732352E3532373031373037323035203135373236392E31343430303739393238342C203132303732382E3938303139363939343935203135373237322E36323931383935393737332C203132303732392E3131363137323837353037203135373237322E37363133313832363337372C203132303734372E3238373933383438313935203135373238392E37363130393839383233332C203132303739382E3539383335303034383333203135373333372E37373436313236313334332C203132303830332E3433323939373037303431203135373334322E3330353334343639322C203132303833302E3331383637393338353337203135373336372E34363338353635333931372C203132303833302E33323530303236343638203135373336372E34363937363333333135322C203132303833352E3035333933333236353237203135373337312E38373935373538393835382C203132303834372E3632363935313333343738203135373338332E37373332393533353632382C203132303930382E3534313836313737373732203135373434312E33363831353633393636372C203132303934382E3838383635353132303632203135373437392E34393636323130393630342C203132303938312E3638303130313333383938203135373531302E393336363235393131342C203132313030382E3738313538323539303235203135373533362E39333631323832353734352C203132313031352E39313733343138333239203135373534332E37373838353634373731382C203132313031372E3332313830333232363832203135373534352E31323532393838363531372C203132313031382E3434393235323539303137203135373534362E32303537323936333138322C203132313034322E3234333936323535373232203135373536392E303135343531363031382C203132313034322E33393136333934373038203135373536392E313531343332393131382C203132313035362E3336373633393436363837203135373538312E353131343332393131322C203132313035362E3637353931383938323933203135373538312E37363239333539383039352C203132313037322E3439363931383938353236203135373539332E36353339333539383335372C203132313037322E3831303236323539313435203135373539332E38373132353334353730342C203132313039302E3535323236323539303134203135373630352E31393932353334353439322C203132313039302E3731313333373831353332203135373630352E32393636393138343538372C203132313131342E35303033333738313234203135373631392E32363436393138343830382C203132313131342E3639313935323739313234203135373631392E33373136313733323733332C203132313132322E3336333935323739373038203135373632332E34333436313733323436322C203132313132322E3337303036383933353735203135373632332E34333738353039343238342C203132313133382E33383530363839323839203135373633312E38393038353039343434342C203132313133382E38333935333939333236203135373633322E31303233313733373232322C203132313135372E3539373533393933333931203135373633392E37313133313733373333322C203132313135372E383637393437383832203135373633392E38313230323036323736342C203132313138322E3837353330323139363538203135373634382E33313138303131363433382C203132313230322E3330343635363532313938203135373635342E39323135383135303832342C203132313230322E3331363033353536303038203135373635342E393235343337333539322C203132313231362E3435303732393731383436203135373635392E363936313333333239382C203132313231382E3333363736363338303039203135373636302E33373538343736373637372C203132313231382E3432313036383732373435203135373636302E34303533383135303032362C203132313235382E3232313334393634333331203135373637332E393530373735343337312C203132313235392E3934333332363430333636203135373637342E35343036353637343833322C203132313331392E3733313938373036393239203135373639352E32323835313931333830342C203132313333302E3139383232313930393531203135373639392E343837343339373638352C203132313334302E3438373733373534373131203135373730352E31383935343234333735382C203132313334392E3239363332323935353337203135373731322E343734393332353732322C203132313334392E3433363833313637323632203135373731322E35393039323538333139362C203132313335362E3236303238373233303136203135373731382E323936303837333935352C203132313336322E3337363230343738373136203135373732342E313335393432353039332C203132313336372E3234353538363738343433203135373733302E31363535323138393238362C203132313337312E3131363530353832323131203135373733372E333931393330393831372C203132313337312E3636313738393337323337203135373733382E32303634323538313236382C203132313337322E3335353439353531303937203135373733382E38393838393038313034342C203132313337332E3137303936353530303033203135373733392E34343237313439333334352C203132313337342E3037363836313238353138203135373733392E38313639393933323536372C203132313337352E3033383336393739393435203135373734302E30303733363034343738342C203132313337362E3031383534303830393836203135373734302E30303634383238323837332C203132313337362E3937393730363839343134203135373733392E38313434303031393437372C203132313337372E3838343933303937383732203135373733392E34333834393431373338372C203132313337382E3639393432353830393637203135373733382E38393332313036323336322C203132313337392E3339313839303830373431203135373733382E3139393530343438352C203132313337392E3933353731343933303434203135373733372E33383430333434393539352C203132313338302E3330393939393332323636203135373733362E34373831333837313037382C203132313338302E3530303336303434343831203135373733352E353136363330313936352C203132313338302E34393934383238323537203135373733342E353336343539313836312C203132313338302E3330373430303139313736203135373733332E35373532393331303138342C203132313337392E3933313439343137303836203135373733322E36373030363930313732372C203132313337352E38333834393431373631203135373732352E303239303639303138333829291A2F687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C23776B744C69746572616C86011283010A2412220A20373630343563303738343364616636646461623338626130663632666561363112330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2347656F6D6574727997011294010A2412220A20353365306632333533623862353337616239316230336462303239393439323912300A2E0A2C687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F7265236E6F746174696F6E1A3A1A380A0831303831303430301A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449647C127A0A2412220A203533653066323335336238623533376162393162303364623032393934393239122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D7323736368656D614167656E63791A261A240A1B68747470733A2F2F6769706F642E766C61616E646572656E2E626512056E6C2D626588011285010A2412220A20353365306632333533623862353337616239316230336462303239393439323912330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73234964656E74696669657289011286010A2412220A20646435303437323837306535636162616339313334353437626331623931303112330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A290A270A25687474703A2F2F7777772E77332E6F72672F6E732F6F7267234F7267616E697A6174696F6E7312710A2412220A20646435303437323837306535636162616339313334353437626331623931303112310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A161A140A1047656D65656E7465204265726C616172480197011294010A2412220A20343265353162653434646565613637623030666435333831363339353865656212270A250A23687474703A2F2F646174612E6575726F70612E65752F6D38672F737461727454696D651A431A410A14323032322D30352D32375430373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D658A011287010A2412220A20343265353162653434646565613637623030666435333831363339353865656212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A2A0A280A26687474703A2F2F646174612E6575726F70612E65752F6D38672F506572696F644F6654696D6595011292010A2412220A20343265353162653434646565613637623030666435333831363339353865656212250A230A21687474703A2F2F646174612E6575726F70612E65752F6D38672F656E6454696D651A431A410A14323032322D30352D32375431373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65', '2024-06-26 12:47:02.671000', '2bc590c1-416e-4e20-8aaa-329ab2dfeb30', 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400'); -INSERT INTO members (member_id, subject, old_id, collection_id, is_in_event_source, member_model, timestamp, +INSERT INTO members (member_id, subject, collection_id, is_in_event_source, member_model, timestamp, transaction_id, version_of) VALUES (2, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600001', - 'mobility-hindrances/https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600001', 1, true, E'\\xB70112B4010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303031122F0A2D0A2B687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F636F6E7472696275746F721A2412220A203965653530313363633463633531636464383332633633323033323030646565B60112B3010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303031122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D732376657273696F6E4E6F7465731A281A260A1D4D6F62696C69747948696E6472616E63655A6F6E65576173416464656412056E6C2D6265D40112D1010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112250A230A21687474703A2F2F7075726C2E6F72672F64632F7465726D732F6D6F6469666965641A4B1A490A1C323032322D30352D32305430393A35383A31352E383634363433335A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65DD0112DA010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303031122D0A2B0A29687474703A2F2F7777772E77332E6F72672F6E732F70726F762367656E657261746564417454696D651A4C1A4A0A1D323032342D30362D32365431303A34373A30332E3937353239303630301A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65C20112BF010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303031123A0A380A3668747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423496E6E616D652E7374617475731A2412220A203566343765663465373861313939366538303666313837303636343666393237D50112D2010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A3E0A3C0A3A68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974234D6F62696C69746569747368696E646572B90112B6010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974237A6F6E651A2412220A206436323839343335373765666238326136336438386563326561343462353936D30112D0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112240A220A20687474703A2F2F7075726C2E6F72672F64632F7465726D732F637265617465641A4B1A490A1C323032322D30352D32335430393A35383A31352E383631303839365A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65E00112DD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F697356657273696F6E4F661A540A520A5068747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F3130383130343030BE0112BB010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236265686565726465721A2412220A203965653530313363633463633531636464383332633633323033323030646565B00112AD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73236964656E7469666965721A2412220A203132323534363263343762333635326565323536333839356465376636363761CA0112C7010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112300A2E0A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449641A361A340A0831303831303430301A28687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D6123696E7465676572BC0112B9010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112340A320A3068747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423706572696F64651A2412220A2063303433633862353431363336346430313631386534313534303830363336329101128E010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112260A240A22687474703A2F2F7075726C2E6F72672F64632F7465726D732F737562737472696E671A071A050A012648019E01129B010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303112280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F6465736372697074696F6E1A121A100A0C6F6D73636872696A76696E674801B30112B0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303031122B0A290A27687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F63726561746F721A2412220A20396565353031336363346363353163646438333263363332303332303064656597011294010A2412220A20633034336338623534313633363464303136313865343135343038303633363212270A250A23687474703A2F2F646174612E6575726F70612E65752F6D38672F737461727454696D651A431A410A14323032322D30352D32375430373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D658A011287010A2412220A20633034336338623534313633363464303136313865343135343038303633363212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A2A0A280A26687474703A2F2F646174612E6575726F70612E65752F6D38672F506572696F644F6654696D6595011292010A2412220A20633034336338623534313633363464303136313865343135343038303633363212250A230A21687474703A2F2F646174612E6575726F70612E65752F6D38672F656E6454696D651A431A410A14323032322D30352D32375431373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D657212700A2412220A20316566303034366266643230303138333737333561363465336566306139353412310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A151A130A0A48696E6465725A6F6E6512056E6C2D626589011286010A2412220A20396565353031336363346363353163646438333263363332303332303064656512330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A290A270A25687474703A2F2F7777772E77332E6F72672F6E732F6F7267234F7267616E697A6174696F6E7312710A2412220A20396565353031336363346363353163646438333263363332303332303064656512310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A161A140A1047656D65656E7465204265726C61617248018B011288010A2412220A20653835363738313339646638613339366239303166333136666461333139626412310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A2D1A2B0A2242657065726B746520646F6F7267616E6720766F6F7220766F657467616E6765727312056E6C2D626587011284010A2412220A20643632383934333537376566623832613633643838656332656134346235393612360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E652E747970651A2412220A2031656630303436626664323030313833373733356136346533656630613935349101128E010A2412220A20643632383934333537376566623832613633643838656332656134346235393612330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E657612740A2412220A20643632383934333537376566623832613633643838656332656134346235393612260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2367656F6D657472791A2412220A20356631343838313838663461623638316436646363613165316636616633623284011281010A2412220A20643632383934333537376566623832613633643838656332656134346235393612330A310A2F68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236765766F6C671A2412220A20653835363738313339646638613339366239303166333136666461333139626480960112FC95010A2412220A203566313438383138386634616236383164366463636131653166366166336232122E0A2C0A2A687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C236173574B541AA295011A9E95010AE994013C687474703A2F2F7777772E6F70656E6769732E6E65742F6465662F6372732F455053472F392E392E312F33313337303E20504F4C59474F4E2028283132313337352E38333834393431373631203135373732352E30323930363930313833382C203133313337352E33323039313439333632203136373732342E32343835373332383630382C203132313336392E3935323931343933353438203135373731372E363031353733323836322C203132313336392E3531353937363937333239203135373731372E31323637393334323037362C203132313336332E3034363937363936393035203135373731302E39343937393334313936382C203132313336322E38303132303432303131203135373731302E37333031333734393031342C203132313335352E38333932303432303336203135373730342E39303931333734393135342C203132313335352E3831353036393533353838203135373730342E383839303836353738312C203132313335352E3636373837333938333538203135373730342E373637353736313533322C203132313334362E35303336373730343931203135373639372E313838303637343330332C203132313334352E37343035373034373534203135373639362E36363736333637343632382C203132313333342E3738333537303437333134203135373639302E35393536333637343536352C203132313333342E3234343534383433323137203135373639302E33333737343939333131382C203132313332332E3337373534383433333438203135373638352E39313537343939323930362C203132313332332E3132373937333232303437203135373638352E38323138363930343033342C203132313236332E3230353937333232323038203135373636352E30383738363930343239372C203132313236332E31393133363632313639203135373636352E30383238343030333231382C203132313236312E3435373336363231323037203135373636342E34383838343030333136372C203132313236312E3434373933313236373834203135373636342E34383536313834393836342C203132313232312E363835323134333534203135373635302E393533303038383733372C203132313231392E3739333233333631353134203135373635302E323731313532333233312C203132313231392E3639363936343433363436203135373635302E323337353632363433332C203132313230352E3531393635363237303137203135373634352E34353234383337333134372C203132313138362E3039353334333437363533203135373633382E38343434313834393433382C203132313136312E3232323730393539333933203135373633302E333930343238303535312C203132313134322E3833313030323232303739203135373632322E3933303031313039312C203132313132372E3034303939303235393337203135373631342E35393537363337343833372C203132313131392E34363930323831303638203135373631302E353835373432363038392C203132313039352E3835353130383739313139203135373539362E37323035343333353332342C203132313037382E3335323535353639343132203135373538352E35343534323635393835382C203132313036322E38343332323738343937203135373537332E38383836373831333639352C203132313034392E3039313536343136303637203135373536312E37323730373531353738332C203132313032352E3336393033373434383237203135373533382E39383635343833393835332C203132313032342E32343138323730353833203135373533372E393036333436363935312C203132313032322E38333834323735303239203135373533362E35363039323233333730382C203132313031352E3730343033373836373431203135373532392E373139353037363334372C203132303938382E3630323431373431333039203135373530332E37313938373137343537362C203132303935352E3739363337393738313233203135373437322E323635383736353839382C203132303935352E3737303232303035323538203135373437322E32343039373536393438352C203132303931352E3431313637393232323933203135373433342E31303134303936323332382C203132303835342E3439383539333438303237203135373337362E35303832373430343735322C203132303834312E3931333034383636343333203135373336342E36303237303436343230322C203132303834312E38383639393733353037203135373336342E353738323336363638392C203132303833372E3134383136313533353832203135373336302E31353931383733323836322C203132303831302E3236393636323232343639203135373333352E30303733393838383839332C203132303830352E34333530303239333033203135373333302E34373636353533303935332C203132303735342E3132303239363032363936203135373238322E34353931323034353034352C203132303733362E3031373036373835323732203135373236352E35323334353637393831372C203132303733322E3633303839323933313135203135373236322E313035393031313539372C203132303732332E3536333835343038313434203135373235322E393533383939343330392C203132303732322E3034323333303639393931203135373235312E34313133343533383237342C203132303731302E3732303035323035383538203135373233372E39353836383830343931342C203132303730312E36353938373539303734203135373232352E313237333330333331312C203132303639342E35323531373534373331203135373231322E383132323830363638322C203132303639342E31353030353031353738203135373231322E303839393436333338322C203132303636362E3938373738333437333634203135373135392E35393332393633333532352C203132303632382E3830303239373638393537203135373038352E38323633353737303031332C203132303631342E3730363533303136303639203135373035382E363038383036353730342C203132303630382E3634353031383839353834203135373034362E38393037353133383630332C203132303630382E3633353532383136313835203135373034362E38373234393631323836342C203132303539372E3231333634383831373735203135373032352E30313237323730353737372C203132303538332E3135383937323836333934203135363939382E31303733343733383734342C203132303537372E3038393137363230313233203135363938362E34383237333638313037352C203132303534362E3136363735313434383937203135363932372E323934393233353130382C203132303532392E3839323836333630333737203135363839362E313338313338323239362C203132303532362E3437383130383232353532203135363838392E363035363932363138322C203132303532362E3436363638363234333235203135363838392E35383339373233303435362C203132303531352E3334393638363234343536203135363836382E353638393732333037372C203132303530392E3535323431353732333034203135363835372E36313534363039353531362C203132303438372E3634363634383236303237203135363831362E32303839303035303435342C203132303438332E3530323730313031303935203135363830382E33373534353138393039382C203132303437392E3532343432393832353036203135363830302E3737363339343734312C203132303436382E3639313137373038363232203135363738302E30353038323433353530372C203132303436372E38343435333831353436203135363737382E343332363032333335322C203132303436372E3036383739323836373639203135363737362E39343930383934343130372C203132303434352E3634393539303634313631203135363733362E30313137303236323331362C203132303433342E35363631363337303435203135363731342E383132363234303637342C203132303433332E3631343432353638343333203135363731322E39383337333836303933332C203132303433302E3739343234303839333939203135363730362E383832383134363633392C203132303432382E35393537313533393733203135363730302E34393331313730343138362C203132303432362E3939303037353734383533203135363639332E3737363136303030352C203132303432362E3031373933343736323631203135363638362E37383737383536313336352C203132303432352E3732343336393430323135203135363637392E35373731343530383838372C203132303432352E39383436393835333539203135363637342E3634353139353338322C203132303432362E3132303535393934393738203135363637322E31393531333038393030332C203132303432372E3234393735363234393036203135363636342E35393938353033353338372C203132303432382E3330353136323637373136203135363635392E323134313434363131312C203132303433302E3035353238373033373636203135363635302E33333334313239363933342C203132303433312E3538373535343435333336203135363634342E38373936363136353433332C203132303433342E3737343438363238363535203135363633332E35343039303432303934382C203132303433342E3832313536313831313036203135363633332E333630353737383931352C203132303433382E3336383536313830393435203135363631382E36353735373738383938382C203132303433382E3434363534343839313335203135363631382E323636353230353438372C203132303433382E3439333534343838393734203135363631372E393639353230353530332C203132303433382E3439373636373631313732203135363631372E39343330303738363635362C203132303433382E3538323034373330343535203135363631372E333930363135323434312C203132303433392E3430303338313438323136203135363631322E31383332313535363630342C203132303434302E3139373930323931313233203135363630372E30383538393031383736382C203132303434302E3230303038333231333332203135363630372E30373138323337373837342C203132303434302E3335363038333230393039203135363630362E303535383233373736312C203132303434302E34303334373632323138203135363630352E363231323332393433382C203132303434312E3036373437363232363334203135363539352E34303332333239343533332C203132303434312E3037353036393031373734203135363539342E39303738323339323637372C203132303434302E3737393036393031333932203135363538362E32363638323339323738372C203132303434302E3733323339333930373635203135363538352E373335343333313830352C203132303433392E3639323339333930383534203135363537382E343037343333313738392C203132303433392E3537373434333336303631203135363537372E38333738303231303036362C203132303433372E3934353434333336313333203135363537312E36333438303231303237382C203132303433372E3835373533303938343933203135363537312E333338323538363133382C203132303433312E303231353330393832203135363535302E363530323538363132382C203132303433302E3937313537323632343731203135363535302E353036343534373031312C203132303432322E3634333631313130373636203135363532372E36363235363032343436372C203132303431382E3331333634393434393138203135363531352E37383336363534343633362C203132303431382E33303439343131313138203135363531352E37353939363130353432352C203132303431342E36393735363337343835203135363530362E30313636343237343635322C203132303431322E33393531383531383033203135363439392E37383433323439313337352C203132303431302E3930313136353632373233203135363439352E37343239313231333637322C203132303430362E3130353331313430333835203135363438322E33373436313832353030362C203132303339372E3930383838353934343832203135363435392E35343534333237353032352C203132303339372E38383839333430343737203135363435392E34393038363036333639362C203132303338352E3332363933343034383732203135363432352E37343038363036333639362C203132303338332E3933353435303831373336203135363432322E30303535363239303734332C203132303338332E3837363534353738313832203135363432312E383534393331393237362C203132303337362E32373535343537373833203135363430332E33303739333139323932322C203132303337362E3138333634343733333734203135363430332E30393735383139323438372C203132303337302E3635323634343733373937203135363339312E31393035383139323333352C203132303337302E3236333938353438393239203135363339302E353032323138333733392C203132303336342E3333333938353438393539203135363338312E37303532313833373535312C203132303336342E30343432323635363538203135363338312E33313733303938333538342C203132303335372E3134353232363536313837203135363337322E39353833303938333437342C203132303335362E3833333138333733393338203135363337322E363134313337313433352C203132303335322E3535373138333733383834203135363336382E33313731333731343531322C203132303335302E3330313334373231373336203135363336362E303531323936373039332C203132303334392E3834393835303336383733203135363336352E363439353634373539392C203132303334322E30303538353033363435203135363335392E343737353634373631352C203132303334312E3437313135353237313635203135363335392E31313033373834333331362C203132303333332E31383235303031323831203135363335342E313737333434393133322C203132303333322E3732323832353632323834203135363335332E39303133343337363937342C203132303333322E3530333137383530383132203135363335332E373736383935343330312C203132303332332E3830373137383531333235203135363334392E313335383935343237352C203132303332332E3433343937383936343631203135363334382E393536363031333839332C203132303331392E3439393937383936323232203135363334372E32353736303133393230382C203132303331332E3337323037343837363832203135363334342E36313332313131343134362C203132303331322E3834363433353237383734203135363334342E343230353137313531392C203132303330342E3136363433353237393034203135363334312E37373935313731353330322C203132303330332E3731323636353135353835203135363334312E36363433363037303831382C203132303330312E3334353636353134393731203135363334312E31383033363037303730382C203132303330312E3332373730303038323339203135363334312E313736373231353335362C203132303239332E3737323730303038323639203135363333392E363630373231353336372C203132303239332E3337383237393735353938203135363333392E35393738343634393232322C203132303238342E363931323739373537203135363333382E35363638343634393237332C203132303238342E3533323439343231313938203135363333382E353530353637303031372C203132303238312E3435343439343231303337203135363333382E32383435363639393930362C203132303238312E3434323332343230323034203135363333382E32383335333032343339332C203132303236382E3739333332343230353536203135363333372E32323135333032343439352C203132303236382E3338393533383431383632203135363333372E323034303231313336352C203132303235322E3232353533383432313534203135363333372E31353730323131333831322C203132303235322E3039383732323639363932203135363333372E31353832363037373938332C203132303233312E323632373232363934203135363333372E363236323630373738332C203132303233312E3139303738313732363638203135363333372E36323833393437383936372C203132303231332E3337303738313732363338203135363333382E323835333934373931322C203132303231332E3230323238343134383331203135363333382E323934343536333634382C203132303139382E3339373238343134383631203135363333392E33343134353633363331382C203132303139382E3230313830333130333833203135363333392E333539313432383432372C203132303138382E3134373830333130363136203135363334302E343638313432383433382C203132303138382E3131373538303430353531203135363334302E343731353639363135392C203132303138362E33373435383034303438203135363334302E363734353639363133382C203132303138362E3231333434353636393734203135363334302E36393539393635323136322C203132303137312E3639303434353636373833203135363334322E38363739393635323337342C203132303137312E3532393339313734303237203135363334322E38393437373832393336322C203132303136362E3036383339313733373335203135363334332E38393437373832393336322C203132303134312E3433313437333338343935203135363334382E34323634393734323332342C203132303134312E3336343630363237373737203135363334382E34333932363831383030322C203132303131302E3034343630363237373437203135363335342E363432323638313737392C203132303130332E3633393034333239393036203135363335352E39313335373739373235362C203132303039392E3634353434393636303132203135363335362E37303534393733393130392C203132303039392E3432353333313533363538203135363335362E37353433323838383837382C203132303038312E3438373333313533353537203135363336312E31363033323838383832382C203132303038312E3334333436323035393837203135363336312E3139373934333731372C203132303036382E3738303436323035383836203135363336342E363832393433373136342C203132303036382E3639363838353337303938203135363336342E37303639313235353138362C203132303035342E3732303838353337343932203135363336382E38343639313235353234362C203132303035342E3135323738383635363838203135363336392E30353332393733343435372C203132303034382E3233383738383635323335203135363337312E36313632393733343535382C203132303034382E3233333233373037303234203135363337312E36313837303732373334322C203132303033382E3130303233373036383932203135363337362E303234373037323732392C203132303033372E39393838393834373832203135363337362E30373031313536383037352C203132303032322E3836303330373731373338203135363338332E303536333931363030312C203132303030332E3834373533343237303535203135363339312E313834353036373231322C203132303030332E3733333439333934313032203135363339312E32333439353130303833342C203131393939392E33323734393339333738203135363339332E32343939353130303532312C203131393939392E33313532383334303236203135363339332E32353535353531303837372C203131393939362E35373534323130343839203135363339342E35313734393137313535322C203131393939352E3735373031353330353533203135363339342E38393433373630313036332C203131393939342E32383130313431383535203135363339352E35373035383232363530362C203131393939332E3639383235353934313532203135363339352E37383032353538343635362C203131393939332E3537363436393234323736203135363339352E38323538373135323730382C203131393939312E3136343538383135383038203135363339362E37363531393537383938332C203131393938382E3733303333323537303134203135363339372E35353938363132323831362C203131393938382E3539373133373931333034203135363339372E36303534323734303430382C203131393938352E34373336373837333237203135363339382E37323333323739343336352C203131393937392E313336353232393139203135363430302E373937323730393730332C203131393937342E3038383633323838373432203135363430322E31333334343835333230382C203131393937322E3739343635393438343835203135363430322E34373539323333383935372C203131393937302E3034303436393739313232203135363430322E383430313538323437362C203131393937302E3033323934323731373638203135363430322E38343131353935303038322C203131393936362E3533333333323631373838203135363430332E33303933383331393939362C203131393936302E35313530343736313534203135363430332E383036343832393230352C203131393935352E3638323236323638343034203135363430342E30323931323634313635342C203131393934392E3932373739313837363434203135363430332E38383534393438323232332C203131393933362E3334333130323734313234203135363430312E38303430373838323334322C203131393933332E3136373530343937343534203135363430312E30373933333736383533342C203131393933332E3136313636393935323032203135363430312E30373830303936373832382C203131393932352E3339383931353130343236203135363339392E333136313530363536372C203131393931312E3432333939343233373738203135363339362E30333431393933363834342C203131393839372E3039383236363230343231203135363339322E333833373038303234342C203131393838372E3536303836303530343637203135363338392E35333937303138343934362C203131393838322E3834383433363831383431203135363338382E303032363738323934372C203131393837392E3438393430333536383132203135363338362E38383230303532373731382C203131393837392E3436333636313533363331203135363338362E38373334393434363434342C203131393837352E3931393631313339393936203135363338352E373132343132363737352C203131393837302E3037393131393930323437203135363338332E34343431353630393135342C203131393836392E3937383436373038323536203135363338332E343036333036333230352C203131393836302E3138333935313837323734203135363337392E38343238393030383834352C203131393834382E3131363031313238313633203135363337342E39313233373334343336332C203131393832332E3438393836343837343931203135363336342E35373735343437313832322C203131393832332E33383936323335383839203135363336342E353336373436383035382C203131393831372E31373039353132393431203135363336322E30383338373630363137382C203131393831362E3032333635373736383337203135363336312E363331313534383338392C203131393739362E3736333033363038313438203135363335342E30323733303431383939382C203131393739322E3932313639323739393935203135363335322E35313130363030333636332C203131393738342E3636373431343032393537203135363334382E353235333137363336332C203131393737392E3232323837353736313638203135363334332E39323636313635313436382C203131393737382E37313931363636323736203135363334332E32383330353633323735382C203131393737382E3034323239323836363738203135363334322E35373431323930343531382C203131393737372E3234303132303139333636203135363334322E303130383735313231372C203131393737362E33343334373536353432203135363334312E36313439343030383935322C203131393737352E3338363831363739353233203135363334312E34303135333935303938382C203131393737342E3430363930373438303138203135363334312E33373838373432343734332C203131393737332E34343134303530373435203135363334312E35343738313533313534352C203131393737322E3532373431333239343137203135363334312E39303138373034303334342C203131393737312E3730303035363333303735203135363334322E34323734333333373239342C203131393737302E3939313132393034383334203135363334332E313034333037313333382C203131393737302E3432373837353132343835203135363334332E393036343739383036392C203131393737302E3033313934303039323637203135363334342E38303331323433343633362C203131393736392E3831383533393531333036203135363334352E37353937383332303533342C203131393736392E3739353837343235303539203135363334362E37333936393235323033382C203131393736392E39363438313533313836203135363334372E37303531393439323630362C203131393737302E33313838373034303636203135363334382E36313931383637303633382C203131393737302E3834343433333337363131203135363334392E34343635343336363938322C203131393737312E3636353633333337343337203135363335302E34393537343336373139332C203131393737322E3337363634393131323331203135363335312E323333373732323338362C203131393737382E3639373634393131343633203135363335362E35373237373232333634322C203131393737392E3734393834383530313531203135363335372E32353535363230373439352C203131393738382E3733393834383439363134203135363336312E35393635363230373638332C203131393738392E3037383234363438363734203135363336312E37343438303733353530382C203131393739332E3039313130353135343038203135363336332E33323837353135363539352C203131393831322E33353139363339323036203135363337302E39333236393538313039322C203131393831332E3530303732303830373437203135363337312E33383539393435343632332C203131393831392E3637303033343938363234203135363337332E38313933393637303538332C203131393834342E32363831333531323634203135363338342E313432343535323834332C203131393834342E3331313932383237323737203135363338342E31363035393032353330342C203131393835362E3439313932383237323437203135363338392E31333638393032353034362C203131393835362E3637333533323932203135363338392E32303639393336373838382C203131393836362E3530393030323939313134203135363339322E3738353331303030372C203131393837322E3432343838303039383139203135363339352E303832383433393037332C203131393837322E3637383333383436303839203135363339352E31373335303535333339352C203131393837362E3333373435353930333531203135363339362E33373232383530323530352C203131393837392E3639393539363432373437203135363339372E34393339393437323131322C203131393837392E3733313536353432393338203135363339372E35303435343130363235322C203131393838342E3532303536353433333931203135363339392E30363635343130363532332C203131393838342E3634323139373334323031203135363339392E31303435303532393234362C203131393839342E33333731393733343233203135363430312E39393535303532393133362C203131393839342E3533313335313430343334203135363430322E30343931363639353636352C203131393930392E3030303335313430313132203135363430352E37333631363639353536342C203131393930392E3039313837313233353735203135363430352E37353835373139343238362C203131393932332E3133303837313233323833203135363430392E30353535373139343439382C203131393932332E3136373333303034343136203135363430392E30363339393033323236342C203131393933302E3934353431323132353531203135363431302E383239333238303630392C203131393933342E3239343439353032323936203135363431312E35393336363233313530382C203131393933342E36343937343830303936203135363431312E36363133323432393434332C203131393934382E3732373734383031313231203135363431332E38313833323432393232322C203131393934392E3336303233383835333937203135363431332E383734343433323231362C203131393935352E3631303233383835333937203135363431342E30333034343332323438322C203131393935352E39363531303239333835203135363431342E30323637303234353838352C203131393936312E3036363130323934323032203135363431332E37393137303234353537322C203131393936312E3234373538393533373537203135363431332E37383030333036303738362C203131393936372E3438323538393533363937203135363431332E3236353033303631312C203131393936372E3733343035373238363937203135363431332E323337383430343939312C203131393937312E33353532393431323632203135363431322E37353333343339383039332C203131393937342E3432313533303231333733203135363431322E333437383431373439372C203131393937352E30343532393639343031203135363431322E32323435373030343033362C203131393937362E3634373239363933383139203135363431312E38303035373030343136342C203131393938312E3833343433373138383337203135363431302E34323735333239313936322C203131393938322E3131303137313433333335203135363431302E33343539393334353731362C203131393938382E3634393137313433303433203135363430382E32303539393334353635362C203131393938382E3737383836323038313534203135363430382E31363135373235393638342C203131393939312E3930303538353531303634203135363430372E303434323933323930372C203131393939342E3430303636373432373132203135363430362E32323831333837373433342C203131393939342E3636333533303735393332203135363430362E31333431323834373338322C203131393939372E3134343933343035323337203135363430352E31363737323832393139372C203131393939372E38363437343430353935203135363430342E39303837343431353237352C203131393939382E3235343532343631313432203135363430342E37343936363733303336342C203131393939392E3932363532343630393831203135363430332E39383336363733303437342C203131393939392E3933353434303736303431203135363430332E39373935373139323430352C203132303030302E3735383434303736363739203135363430332E363030353731393233342C203132303030332E3439323631353039363331203135363430322E333431323535313339342C203132303030372E3833353830303433373334203135363430302E33353439383232323932342C203132303032362E3835363436353733333039203135363339322E323233343933323738362C203132303032362E3938363130313532313231203135363339322E31363538383433313639362C203132303034322E3133383638343335383137203135363338352E31373331353132323432372C203132303035322E3231373938373838313834203135363338302E37393034393933363235372C203132303035372E3835313431313631313437203135363337382E33343930393530363533382C203132303037312E3439353433303438353139203135363337342E33303734333532303833382C203132303038332E3934343838323930393638203135363337302E383533393333353231382C203132303130312E37303131363030343335203135363336362E343932353638393836332C203132303130352E3538343535303334343035203135363336352E373232353032363038332C203132303131312E3938383337383434303032203135363336342E34353135333637393432382C203132303134332E3237343030333634343632203135363335382E32353533343438303539322C203132303136372E38373135363735383139203135363335332E37333038363239323332362C203132303137332E3235303239393935313534203135363335322E373435393237343936332C203132303138372E3631323136313133393439203135363335302E353938303236373530382C203132303138392E3235393331333436303737203135363335302E34303631383937333535342C203132303139392E32303036323939383137203135363334392E33303936313932313238342C203132303231332E3832333534333532353936203135363334382E32373534393632353034382C203132303233312E3532333235353437303931203135363334372E36323239333131313136332C203132303235322E3235393837373330373131203135363334372E31353731363332353835352C203132303236382E3135383231343536393932203135363334372E32303333393037393034372C203132303238302E3539393539303135363032203135363334382E323437393538383038382C203132303238332E3539313938333036393936203135363334382E35303635363036363739372C203132303239322E3030313235333435363238203135363334392E35303435393838353839342C203132303239392E33353133313430343833203135363335302E39373934373533343436382C203132303330312E3437393735313238303935203135363335312E34313436393434343931372C203132303330392E3636373334323530323931203135363335332E393035383732383337332C203132303331352E3533363437333035343332203135363335362E34333835393337373633372C203132303331392E32383132313434383536203135363335382E303535343436353533342C203132303332372E3638333432363536323336203135363336322E353339363533383038382C203132303332382E3034333137343337323333203135363336322E37353536353632333137382C203132303332382E3035393834343732343132203135363336322E37363536323135363838362C203132303333362E3037373536333538393132203135363336372E353337343036303431372C203132303334332E34323832333936363037203135363337332E33323132333733333636342C203132303334352E3436393233343530323734203135363337352E33373132383331353834392C203132303334392E3538303739313538313637203135363337392E35303330333236333434372C203132303335362E3137373439323030393334203135363338372E34393537353839343939382C203132303336312E3735303339313638363531203135363339352E37363330303939353339352C203132303336372E3036353938393831343734203135363430372E323036323938313432322C203132303337342E3539323738333339363032203135363432352E353732323239303033332C203132303337352E3935353330373438383638203135363432392E32323937383832383030332C203132303338382E3530363933313435383637203135363436322E393531393131323731372C203132303339362E3639323930313233303731203135363438352E37353139373434383337352C203132303430312E3439363638383539343234203135363439392E31343233383137353038362C203132303430312E3531333139383836393734203135363439392E31383737313433313939372C203132303430332E3031353030363739353432203135363530332E32353031393437353736322C203132303430352E3331373831343831393232203135363530392E34383336373530383534372C203132303430382E3932323637343738323038203135363531392E32323031393737343435352C203132303431332E3234383335303534393831203135363533312E30383733333435353239382C203132303432312E3535303335333436353436203135363535332E38363030323336353737362C203132303432382E3331333932343538323135203135363537342E33323838323938363135372C203132303432392E3833323530323332333239203135363538302E31303037323739373939372C203132303433302E3739343037353136303635203135363538362E38373631313831333335342C203132303433312E3037323433393334333134203135363539352E30303232383332333334352C203132303433302E3433383731313734323139203135363630342E37353434333437313135362C203132303433302E3331363939363932383837203135363630352E35343731343134363739382C203132303432392E3532313335373234393832203135363631302E36333234343730333039322C203132303432382E3730313631383532333338203135363631352E38343837383434333534382C203132303432382E36393833333233393036203135363631352E38363939393231333434352C203132303432382E3631343335383138303031203135363631362E343139373330323632392C203132303432382E3630303634333630313937203135363631362E35303633393437323636332C203132303432352E3132323239353239363538203135363633302E393234383230323032352C203132303432312E3936303531333731323835203135363634322E31373430393537393033372C203132303432302E3337343337373337383538203135363634372E38313935383039363234342C203132303432302E3238323335313830383837203135363634382E32303532343436393538362C203132303431382E3439333335313830343334203135363635372E32383332343436393734362C203132303431382E3439323332353630323038203135363635372E32383834363636343738352C203132303431372E3431343332353630373933203135363636322E37383934363636343631352C203132303431372E3337353335383034303932203135363636332E30313537323735333838322C203132303431362E3139353335383034313232203135363637302E39353237323735343135332C203132303431362E3134383636393637393536203135363637312E34313131363434303831342C203132303431352E3939393636393637353633203135363637342E30393831363434303731332C203132303431352E3939383935303931323733203135363637342E3131313434353738332C203132303431352E3732353935303931303833203135363637392E323833343435373831342C203132303431352E3732333133383638323536203135363637392E37353033393534393733372C203132303431362E3033363133383638333537203135363638372E34333833393534393833382C203132303431362E30373936383638313633203135363638372E39323339303739343132372C203132303431372E3131383638363832303833203135363639352E33393239303739343137372C203132303431372E3230383030393037303433203135363639352E38363634363235353638352C203132303431382E3932363030393036353138203135363730332E30353334363235353538342C203132303431392E3036313033373337323336203135363730332E35313737363635353232382C203132303432312E3432313033373337313737203135363731302E33373637363635353333382C203132303432312E3631303434313835323734203135363731302E38343739373238323533342C203132303432342E3538363434313835363236203135363731372E32383539373238323633362C203132303432342E3638393632383532393833203135363731372E34393631333334323930332C203132303432352E3639373632383533313134203135363731392E3433333133333432382C203132303432352E3730323035323137353838203135363731392E343431363134323033362C203132303433362E3738383035323137313335203135363734302E363435363134323033352C203132303435382E3230373438363835393839203135363738312E353833343435353132392C203132303435382E3938333230373132353933203135363738332E30363639313035353738332C203132303435392E3832393236393632353238203135363738342E363834303330323733362C203132303437302E3636323832323930383437203135363830352E34313031373536343431322C203132303437342E3634383331383335343034203135363831332E30323330333433383937342C203132303437342E36353833323034353036203135363831332E30343230343033343733382C203132303437382E3830373332303434373039203135363832302E38383530343033343538362C203132303530302E3731333338323937353738203135363836322E32393231353835333736362C203132303530362E3531303534393633383833203135363837332E323435343733353938322C203132303531372E3632313537363131313439203135363839342E323439313831353531342C203132303532312E3032393531333937313233203135363930302E37363835383436333936382C203132303533372E3330333133363339383535203135363933312E393234383631373731342C203132303536382E3232353039323232383233203135363939312E313131373737323738392C203132303537342E3239343832333739343635203135373030322E37333632363331383630332C203132303538382E3335303233303533333935203135373032392E36343330343139393234342C203132303539392E3736373730373639383839203135373035312E34393433383630303933322C203132303630352E3832343938313131303432203135373036332E323034323438363136352C203132303631392E3931393833303633353139203135373039302E343233383930313837372C203132303635382E3130363435393337303438203135373136342E31383931373330313438332C203132303638352E3237303231363532373637203135373231362E36383837303336363235342C203132303638352E3237333638383232313332203135373231362E36393534303130333838332C203132303638352E3730333638383232313032203135373231372E35323334303130343034342C203132303638352E38313436323034393831203135373231372E37323534373936373938322C203132303639332E31313936323034393738203135373233302E33333434373936383039332C203132303639332E3336313537323731373937203135373233302E37313139393936313330342C203132303730322E3637333537323731363936203135373234332E38393939393936313033322C203132303730322E3933323536343939323732203135373234342E32333536333436303437332C203132303731342E3531383536343938383139203135373235382E30303136333436303733352C203132303731342E3738343238333337323439203135373235382E32393331383436333834362C203132303731362E3434383238333337373032203135373235392E39383031383436333734352C203132303731362E3435363031373036393732203135373235392E39383830303739393333352C203132303732352E3532373031373037323035203135373236392E31343430303739393238342C203132303732382E3938303139363939343935203135373237322E36323931383935393737332C203132303732392E3131363137323837353037203135373237322E37363133313832363337372C203132303734372E3238373933383438313935203135373238392E37363130393839383233332C203132303739382E3539383335303034383333203135373333372E37373436313236313334332C203132303830332E3433323939373037303431203135373334322E3330353334343639322C203132303833302E3331383637393338353337203135373336372E34363338353635333931372C203132303833302E33323530303236343638203135373336372E34363937363333333135322C203132303833352E3035333933333236353237203135373337312E38373935373538393835382C203132303834372E3632363935313333343738203135373338332E37373332393533353632382C203132303930382E3534313836313737373732203135373434312E33363831353633393636372C203132303934382E3838383635353132303632203135373437392E34393636323130393630342C203132303938312E3638303130313333383938203135373531302E393336363235393131342C203132313030382E3738313538323539303235203135373533362E39333631323832353734352C203132313031352E39313733343138333239203135373534332E37373838353634373731382C203132313031372E3332313830333232363832203135373534352E31323532393838363531372C203132313031382E3434393235323539303137203135373534362E32303537323936333138322C203132313034322E3234333936323535373232203135373536392E303135343531363031382C203132313034322E33393136333934373038203135373536392E313531343332393131382C203132313035362E3336373633393436363837203135373538312E353131343332393131322C203132313035362E3637353931383938323933203135373538312E37363239333539383039352C203132313037322E3439363931383938353236203135373539332E36353339333539383335372C203132313037322E3831303236323539313435203135373539332E38373132353334353730342C203132313039302E3535323236323539303134203135373630352E31393932353334353439322C203132313039302E3731313333373831353332203135373630352E32393636393138343538372C203132313131342E35303033333738313234203135373631392E32363436393138343830382C203132313131342E3639313935323739313234203135373631392E33373136313733323733332C203132313132322E3336333935323739373038203135373632332E34333436313733323436322C203132313132322E3337303036383933353735203135373632332E34333738353039343238342C203132313133382E33383530363839323839203135373633312E38393038353039343434342C203132313133382E38333935333939333236203135373633322E31303233313733373232322C203132313135372E3539373533393933333931203135373633392E37313133313733373333322C203132313135372E383637393437383832203135373633392E38313230323036323736342C203132313138322E3837353330323139363538203135373634382E33313138303131363433382C203132313230322E3330343635363532313938203135373635342E39323135383135303832342C203132313230322E3331363033353536303038203135373635342E393235343337333539322C203132313231362E3435303732393731383436203135373635392E363936313333333239382C203132313231382E3333363736363338303039203135373636302E33373538343736373637372C203132313231382E3432313036383732373435203135373636302E34303533383135303032362C203132313235382E3232313334393634333331203135373637332E393530373735343337312C203132313235392E3934333332363430333636203135373637342E35343036353637343833322C203132313331392E3733313938373036393239203135373639352E32323835313931333830342C203132313333302E3139383232313930393531203135373639392E343837343339373638352C203132313334302E3438373733373534373131203135373730352E31383935343234333735382C203132313334392E3239363332323935353337203135373731322E343734393332353732322C203132313334392E3433363833313637323632203135373731322E35393039323538333139362C203132313335362E3236303238373233303136203135373731382E323936303837333935352C203132313336322E3337363230343738373136203135373732342E313335393432353039332C203132313336372E3234353538363738343433203135373733302E31363535323138393238362C203132313337312E3131363530353832323131203135373733372E333931393330393831372C203132313337312E3636313738393337323337203135373733382E32303634323538313236382C203132313337322E3335353439353531303937203135373733382E38393838393038313034342C203132313337332E3137303936353530303033203135373733392E34343237313439333334352C203132313337342E3037363836313238353138203135373733392E38313639393933323536372C203132313337352E3033383336393739393435203135373734302E30303733363034343738342C203132313337362E3031383534303830393836203135373734302E30303634383238323837332C203132313337362E3937393730363839343134203135373733392E38313434303031393437372C203132313337372E3838343933303937383732203135373733392E34333834393431373338372C203132313337382E3639393432353830393637203135373733382E38393332313036323336322C203132313337392E3339313839303830373431203135373733382E3139393530343438352C203132313337392E3933353731343933303434203135373733372E33383430333434393539352C203132313338302E3330393939393332323636203135373733362E34373831333837313037382C203132313338302E3530303336303434343831203135373733352E353136363330313936352C203132313338302E34393934383238323537203135373733342E353336343539313836312C203132313338302E3330373430303139313736203135373733332E35373532393331303138342C203132313337392E3933313439343137303836203135373733322E36373030363930313732372C203132313337352E38333834393431373631203135373732352E303239303639303138333829291A2F687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C23776B744C69746572616C86011283010A2412220A20356631343838313838663461623638316436646363613165316636616633623212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2347656F6D6574727997011294010A2412220A20313232353436326334376233363532656532353633383935646537663636376112300A2E0A2C687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F7265236E6F746174696F6E1A3A1A380A0831303831303430301A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449647C127A0A2412220A203132323534363263343762333635326565323536333839356465376636363761122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D7323736368656D614167656E63791A261A240A1B68747470733A2F2F6769706F642E766C61616E646572656E2E626512056E6C2D626588011285010A2412220A20313232353436326334376233363532656532353633383935646537663636376112330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73234964656E74696669657271126F0A2412220A20356634376566346537386131393936653830366631383730363634366639323712310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A141A120A09496E206F706D61616B12056E6C2D6265', '2024-06-26 10:47:03.975000', 'c1fce741-2243-4c62-8707-b734a626cc5a', 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400'); -INSERT INTO members (member_id, subject, old_id, collection_id, is_in_event_source, member_model, timestamp, +INSERT INTO members (member_id, subject, collection_id, is_in_event_source, member_model, timestamp, transaction_id, version_of) VALUES (3, 'https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600002', - 'mobility-hindrances/https://private-api.gipod.beta-vlaanderen.be/api/v1/mobility-hindrances/10810400/600002', 1, true, E'\\xB70112B4010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303032122F0A2D0A2B687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F636F6E7472696275746F721A2412220A206531316539313463633762633834663537613961353834383935313366306563B60112B3010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303032122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D732376657273696F6E4E6F7465731A281A260A1D4D6F62696C69747948696E6472616E63655A6F6E65576173416464656412056E6C2D6265D40112D1010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212250A230A21687474703A2F2F7075726C2E6F72672F64632F7465726D732F6D6F6469666965641A4B1A490A1C323032322D30352D32305430393A35383A31352E383634363433335A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65DD0112DA010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303032122D0A2B0A29687474703A2F2F7777772E77332E6F72672F6E732F70726F762367656E657261746564417454696D651A4C1A4A0A1D323032342D30362D32375430383A34373A30342E3030333638323230301A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65C20112BF010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303032123A0A380A3668747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423496E6E616D652E7374617475731A2412220A206466623239613537393532633132633138393233326630643337396334656161D50112D2010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A3E0A3C0A3A68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974234D6F62696C69746569747368696E646572B90112B6010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974237A6F6E651A2412220A203531653366663663353738313564653963656537303336623434623636353038D30112D0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212240A220A20687474703A2F2F7075726C2E6F72672F64632F7465726D732F637265617465641A4B1A490A1C323032322D30352D32335430393A35383A31352E383631303839365A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D65E00112DD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F697356657273696F6E4F661A540A520A5068747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F3130383130343030BE0112BB010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236265686565726465721A2412220A206531316539313463633762633834663537613961353834383935313366306563B00112AD010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73236964656E7469666965721A2412220A203138343037343364343337613931613464353361393561313238303762323337CA0112C7010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212300A2E0A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449641A361A340A0831303831303430301A28687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D6123696E7465676572BC0112B9010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212340A320A3068747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C697465697423706572696F64651A2412220A2039633839366666363061663930613636323065396434323161393763333166339101128E010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212260A240A22687474703A2F2F7075726C2E6F72672F64632F7465726D732F737562737472696E671A071A050A012648019E01129B010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F36303030303212280A260A24687474703A2F2F7075726C2E6F72672F64632F7465726D732F6465736372697074696F6E1A121A100A0C6F6D73636872696A76696E674801B30112B0010A5B0A590A5768747470733A2F2F707269766174652D6170692E6769706F642E626574612D766C61616E646572656E2E62652F6170692F76312F6D6F62696C6974792D68696E6472616E6365732F31303831303430302F363030303032122B0A290A27687474703A2F2F7075726C2E6F72672F64632F656C656D656E74732F312E312F63726561746F721A2412220A20653131653931346363376263383466353761396135383438393531336630656397011294010A2412220A20396338393666663630616639306136363230653964343231613937633331663312270A250A23687474703A2F2F646174612E6575726F70612E65752F6D38672F737461727454696D651A431A410A14323032322D30352D32375430373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D658A011287010A2412220A20396338393666663630616639306136363230653964343231613937633331663312330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A2A0A280A26687474703A2F2F646174612E6575726F70612E65752F6D38672F506572696F644F6654696D6595011292010A2412220A20396338393666663630616639306136363230653964343231613937633331663312250A230A21687474703A2F2F646174612E6575726F70612E65752F6D38672F656E6454696D651A431A410A14323032322D30352D32375431373A30303A30305A1A29687474703A2F2F7777772E77332E6F72672F323030312F584D4C536368656D61236461746554696D6597011294010A2412220A20313834303734336434333761393161346435336139356131323830376232333712300A2E0A2C687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F7265236E6F746174696F6E1A3A1A380A0831303831303430301A2C68747470733A2F2F6769706F642E766C61616E646572656E2E62652F6E732F6769706F64236769706F6449647C127A0A2412220A203138343037343364343337613931613464353361393561313238303762323337122A0A280A26687474703A2F2F7777772E77332E6F72672F6E732F61646D7323736368656D614167656E63791A261A240A1B68747470733A2F2F6769706F642E766C61616E646572656E2E626512056E6C2D626588011285010A2412220A20313834303734336434333761393161346435336139356131323830376232333712330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A280A260A24687474703A2F2F7777772E77332E6F72672F6E732F61646D73234964656E74696669657289011286010A2412220A20653131653931346363376263383466353761396135383438393531336630656312330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A290A270A25687474703A2F2F7777772E77332E6F72672F6E732F6F7267234F7267616E697A6174696F6E7312710A2412220A20653131653931346363376263383466353761396135383438393531336630656312310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A161A140A1047656D65656E7465204265726C61617248017212700A2412220A20646536663564366634613435376562396133666662383136363661643339356312310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A151A130A0A48696E6465725A6F6E6512056E6C2D626571126F0A2412220A20646662323961353739353263313263313839323332663064333739633465616112310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A141A120A09496E206F706D61616B12056E6C2D626587011284010A2412220A20353165336666366335373831356465396365653730333662343462363635303812360A340A3268747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E652E747970651A2412220A2064653666356436663461343537656239613366666238313636366164333935639101128E010A2412220A20353165336666366335373831356465396365653730333662343462363635303812330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A310A2F0A2D68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974235A6F6E657612740A2412220A20353165336666366335373831356465396365653730333662343462363635303812260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2367656F6D657472791A2412220A20396366333433396266356631316239656532623630633534626237343564396284011281010A2412220A20353165336666366335373831356465396365653730333662343462363635303812330A310A2F68747470733A2F2F646174612E766C61616E646572656E2E62652F6E732F6D6F62696C6974656974236765766F6C671A2412220A20363963333161383563356231343566333834303731613233393066373637333480960112FC95010A2412220A203963663334333962663566313162396565326236306335346262373435643962122E0A2C0A2A687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C236173574B541AA295011A9E95010AE994013C687474703A2F2F7777772E6F70656E6769732E6E65742F6465662F6372732F455053472F392E392E312F33313337303E20504F4C59474F4E2028283132313337352E38333834393431373631203135373732352E30323930363930313833382C203133313337352E33323039313439333632203136373732342E32343835373332383630382C203132313336392E3935323931343933353438203135373731372E363031353733323836322C203132313336392E3531353937363937333239203135373731372E31323637393334323037362C203132313336332E3034363937363936393035203135373731302E39343937393334313936382C203132313336322E38303132303432303131203135373731302E37333031333734393031342C203132313335352E38333932303432303336203135373730342E39303931333734393135342C203132313335352E3831353036393533353838203135373730342E383839303836353738312C203132313335352E3636373837333938333538203135373730342E373637353736313533322C203132313334362E35303336373730343931203135373639372E313838303637343330332C203132313334352E37343035373034373534203135373639362E36363736333637343632382C203132313333342E3738333537303437333134203135373639302E35393536333637343536352C203132313333342E3234343534383433323137203135373639302E33333737343939333131382C203132313332332E3337373534383433333438203135373638352E39313537343939323930362C203132313332332E3132373937333232303437203135373638352E38323138363930343033342C203132313236332E3230353937333232323038203135373636352E30383738363930343239372C203132313236332E31393133363632313639203135373636352E30383238343030333231382C203132313236312E3435373336363231323037203135373636342E34383838343030333136372C203132313236312E3434373933313236373834203135373636342E34383536313834393836342C203132313232312E363835323134333534203135373635302E393533303038383733372C203132313231392E3739333233333631353134203135373635302E323731313532333233312C203132313231392E3639363936343433363436203135373635302E323337353632363433332C203132313230352E3531393635363237303137203135373634352E34353234383337333134372C203132313138362E3039353334333437363533203135373633382E38343434313834393433382C203132313136312E3232323730393539333933203135373633302E333930343238303535312C203132313134322E3833313030323232303739203135373632322E3933303031313039312C203132313132372E3034303939303235393337203135373631342E35393537363337343833372C203132313131392E34363930323831303638203135373631302E353835373432363038392C203132313039352E3835353130383739313139203135373539362E37323035343333353332342C203132313037382E3335323535353639343132203135373538352E35343534323635393835382C203132313036322E38343332323738343937203135373537332E38383836373831333639352C203132313034392E3039313536343136303637203135373536312E37323730373531353738332C203132313032352E3336393033373434383237203135373533382E39383635343833393835332C203132313032342E32343138323730353833203135373533372E393036333436363935312C203132313032322E38333834323735303239203135373533362E35363039323233333730382C203132313031352E3730343033373836373431203135373532392E373139353037363334372C203132303938382E3630323431373431333039203135373530332E37313938373137343537362C203132303935352E3739363337393738313233203135373437322E323635383736353839382C203132303935352E3737303232303035323538203135373437322E32343039373536393438352C203132303931352E3431313637393232323933203135373433342E31303134303936323332382C203132303835342E3439383539333438303237203135373337362E35303832373430343735322C203132303834312E3931333034383636343333203135373336342E36303237303436343230322C203132303834312E38383639393733353037203135373336342E353738323336363638392C203132303833372E3134383136313533353832203135373336302E31353931383733323836322C203132303831302E3236393636323232343639203135373333352E30303733393838383839332C203132303830352E34333530303239333033203135373333302E34373636353533303935332C203132303735342E3132303239363032363936203135373238322E34353931323034353034352C203132303733362E3031373036373835323732203135373236352E35323334353637393831372C203132303733322E3633303839323933313135203135373236322E313035393031313539372C203132303732332E3536333835343038313434203135373235322E393533383939343330392C203132303732322E3034323333303639393931203135373235312E34313133343533383237342C203132303731302E3732303035323035383538203135373233372E39353836383830343931342C203132303730312E36353938373539303734203135373232352E313237333330333331312C203132303639342E35323531373534373331203135373231322E383132323830363638322C203132303639342E31353030353031353738203135373231322E303839393436333338322C203132303636362E3938373738333437333634203135373135392E35393332393633333532352C203132303632382E3830303239373638393537203135373038352E38323633353737303031332C203132303631342E3730363533303136303639203135373035382E363038383036353730342C203132303630382E3634353031383839353834203135373034362E38393037353133383630332C203132303630382E3633353532383136313835203135373034362E38373234393631323836342C203132303539372E3231333634383831373735203135373032352E30313237323730353737372C203132303538332E3135383937323836333934203135363939382E31303733343733383734342C203132303537372E3038393137363230313233203135363938362E34383237333638313037352C203132303534362E3136363735313434383937203135363932372E323934393233353130382C203132303532392E3839323836333630333737203135363839362E313338313338323239362C203132303532362E3437383130383232353532203135363838392E363035363932363138322C203132303532362E3436363638363234333235203135363838392E35383339373233303435362C203132303531352E3334393638363234343536203135363836382E353638393732333037372C203132303530392E3535323431353732333034203135363835372E36313534363039353531362C203132303438372E3634363634383236303237203135363831362E32303839303035303435342C203132303438332E3530323730313031303935203135363830382E33373534353138393039382C203132303437392E3532343432393832353036203135363830302E3737363339343734312C203132303436382E3639313137373038363232203135363738302E30353038323433353530372C203132303436372E38343435333831353436203135363737382E343332363032333335322C203132303436372E3036383739323836373639203135363737362E39343930383934343130372C203132303434352E3634393539303634313631203135363733362E30313137303236323331362C203132303433342E35363631363337303435203135363731342E383132363234303637342C203132303433332E3631343432353638343333203135363731322E39383337333836303933332C203132303433302E3739343234303839333939203135363730362E383832383134363633392C203132303432382E35393537313533393733203135363730302E34393331313730343138362C203132303432362E3939303037353734383533203135363639332E3737363136303030352C203132303432362E3031373933343736323631203135363638362E37383737383536313336352C203132303432352E3732343336393430323135203135363637392E35373731343530383838372C203132303432352E39383436393835333539203135363637342E3634353139353338322C203132303432362E3132303535393934393738203135363637322E31393531333038393030332C203132303432372E3234393735363234393036203135363636342E35393938353033353338372C203132303432382E3330353136323637373136203135363635392E323134313434363131312C203132303433302E3035353238373033373636203135363635302E33333334313239363933342C203132303433312E3538373535343435333336203135363634342E38373936363136353433332C203132303433342E3737343438363238363535203135363633332E35343039303432303934382C203132303433342E3832313536313831313036203135363633332E333630353737383931352C203132303433382E3336383536313830393435203135363631382E36353735373738383938382C203132303433382E3434363534343839313335203135363631382E323636353230353438372C203132303433382E3439333534343838393734203135363631372E393639353230353530332C203132303433382E3439373636373631313732203135363631372E39343330303738363635362C203132303433382E3538323034373330343535203135363631372E333930363135323434312C203132303433392E3430303338313438323136203135363631322E31383332313535363630342C203132303434302E3139373930323931313233203135363630372E30383538393031383736382C203132303434302E3230303038333231333332203135363630372E30373138323337373837342C203132303434302E3335363038333230393039203135363630362E303535383233373736312C203132303434302E34303334373632323138203135363630352E363231323332393433382C203132303434312E3036373437363232363334203135363539352E34303332333239343533332C203132303434312E3037353036393031373734203135363539342E39303738323339323637372C203132303434302E3737393036393031333932203135363538362E32363638323339323738372C203132303434302E3733323339333930373635203135363538352E373335343333313830352C203132303433392E3639323339333930383534203135363537382E343037343333313738392C203132303433392E3537373434333336303631203135363537372E38333738303231303036362C203132303433372E3934353434333336313333203135363537312E36333438303231303237382C203132303433372E3835373533303938343933203135363537312E333338323538363133382C203132303433312E303231353330393832203135363535302E363530323538363132382C203132303433302E3937313537323632343731203135363535302E353036343534373031312C203132303432322E3634333631313130373636203135363532372E36363235363032343436372C203132303431382E3331333634393434393138203135363531352E37383336363534343633362C203132303431382E33303439343131313138203135363531352E37353939363130353432352C203132303431342E36393735363337343835203135363530362E30313636343237343635322C203132303431322E33393531383531383033203135363439392E37383433323439313337352C203132303431302E3930313136353632373233203135363439352E37343239313231333637322C203132303430362E3130353331313430333835203135363438322E33373436313832353030362C203132303339372E3930383838353934343832203135363435392E35343534333237353032352C203132303339372E38383839333430343737203135363435392E34393038363036333639362C203132303338352E3332363933343034383732203135363432352E37343038363036333639362C203132303338332E3933353435303831373336203135363432322E30303535363239303734332C203132303338332E3837363534353738313832203135363432312E383534393331393237362C203132303337362E32373535343537373833203135363430332E33303739333139323932322C203132303337362E3138333634343733333734203135363430332E30393735383139323438372C203132303337302E3635323634343733373937203135363339312E31393035383139323333352C203132303337302E3236333938353438393239203135363339302E353032323138333733392C203132303336342E3333333938353438393539203135363338312E37303532313833373535312C203132303336342E30343432323635363538203135363338312E33313733303938333538342C203132303335372E3134353232363536313837203135363337322E39353833303938333437342C203132303335362E3833333138333733393338203135363337322E363134313337313433352C203132303335322E3535373138333733383834203135363336382E33313731333731343531322C203132303335302E3330313334373231373336203135363336362E303531323936373039332C203132303334392E3834393835303336383733203135363336352E363439353634373539392C203132303334322E30303538353033363435203135363335392E343737353634373631352C203132303334312E3437313135353237313635203135363335392E31313033373834333331362C203132303333332E31383235303031323831203135363335342E313737333434393133322C203132303333322E3732323832353632323834203135363335332E39303133343337363937342C203132303333322E3530333137383530383132203135363335332E373736383935343330312C203132303332332E3830373137383531333235203135363334392E313335383935343237352C203132303332332E3433343937383936343631203135363334382E393536363031333839332C203132303331392E3439393937383936323232203135363334372E32353736303133393230382C203132303331332E3337323037343837363832203135363334342E36313332313131343134362C203132303331322E3834363433353237383734203135363334342E343230353137313531392C203132303330342E3136363433353237393034203135363334312E37373935313731353330322C203132303330332E3731323636353135353835203135363334312E36363433363037303831382C203132303330312E3334353636353134393731203135363334312E31383033363037303730382C203132303330312E3332373730303038323339203135363334312E313736373231353335362C203132303239332E3737323730303038323639203135363333392E363630373231353336372C203132303239332E3337383237393735353938203135363333392E35393738343634393232322C203132303238342E363931323739373537203135363333382E35363638343634393237332C203132303238342E3533323439343231313938203135363333382E353530353637303031372C203132303238312E3435343439343231303337203135363333382E32383435363639393930362C203132303238312E3434323332343230323034203135363333382E32383335333032343339332C203132303236382E3739333332343230353536203135363333372E32323135333032343439352C203132303236382E3338393533383431383632203135363333372E323034303231313336352C203132303235322E3232353533383432313534203135363333372E31353730323131333831322C203132303235322E3039383732323639363932203135363333372E31353832363037373938332C203132303233312E323632373232363934203135363333372E363236323630373738332C203132303233312E3139303738313732363638203135363333372E36323833393437383936372C203132303231332E3337303738313732363338203135363333382E323835333934373931322C203132303231332E3230323238343134383331203135363333382E323934343536333634382C203132303139382E3339373238343134383631203135363333392E33343134353633363331382C203132303139382E3230313830333130333833203135363333392E333539313432383432372C203132303138382E3134373830333130363136203135363334302E343638313432383433382C203132303138382E3131373538303430353531203135363334302E343731353639363135392C203132303138362E33373435383034303438203135363334302E363734353639363133382C203132303138362E3231333434353636393734203135363334302E36393539393635323136322C203132303137312E3639303434353636373833203135363334322E38363739393635323337342C203132303137312E3532393339313734303237203135363334322E38393437373832393336322C203132303136362E3036383339313733373335203135363334332E38393437373832393336322C203132303134312E3433313437333338343935203135363334382E34323634393734323332342C203132303134312E3336343630363237373737203135363334382E34333932363831383030322C203132303131302E3034343630363237373437203135363335342E363432323638313737392C203132303130332E3633393034333239393036203135363335352E39313335373739373235362C203132303039392E3634353434393636303132203135363335362E37303534393733393130392C203132303039392E3432353333313533363538203135363335362E37353433323838383837382C203132303038312E3438373333313533353537203135363336312E31363033323838383832382C203132303038312E3334333436323035393837203135363336312E3139373934333731372C203132303036382E3738303436323035383836203135363336342E363832393433373136342C203132303036382E3639363838353337303938203135363336342E37303639313235353138362C203132303035342E3732303838353337343932203135363336382E38343639313235353234362C203132303035342E3135323738383635363838203135363336392E30353332393733343435372C203132303034382E3233383738383635323335203135363337312E36313632393733343535382C203132303034382E3233333233373037303234203135363337312E36313837303732373334322C203132303033382E3130303233373036383932203135363337362E303234373037323732392C203132303033372E39393838393834373832203135363337362E30373031313536383037352C203132303032322E3836303330373731373338203135363338332E303536333931363030312C203132303030332E3834373533343237303535203135363339312E313834353036373231322C203132303030332E3733333439333934313032203135363339312E32333439353130303833342C203131393939392E33323734393339333738203135363339332E32343939353130303532312C203131393939392E33313532383334303236203135363339332E32353535353531303837372C203131393939362E35373534323130343839203135363339342E35313734393137313535322C203131393939352E3735373031353330353533203135363339342E38393433373630313036332C203131393939342E32383130313431383535203135363339352E35373035383232363530362C203131393939332E3639383235353934313532203135363339352E37383032353538343635362C203131393939332E3537363436393234323736203135363339352E38323538373135323730382C203131393939312E3136343538383135383038203135363339362E37363531393537383938332C203131393938382E3733303333323537303134203135363339372E35353938363132323831362C203131393938382E3539373133373931333034203135363339372E36303534323734303430382C203131393938352E34373336373837333237203135363339382E37323333323739343336352C203131393937392E313336353232393139203135363430302E373937323730393730332C203131393937342E3038383633323838373432203135363430322E31333334343835333230382C203131393937322E3739343635393438343835203135363430322E34373539323333383935372C203131393937302E3034303436393739313232203135363430322E383430313538323437362C203131393937302E3033323934323731373638203135363430322E38343131353935303038322C203131393936362E3533333333323631373838203135363430332E33303933383331393939362C203131393936302E35313530343736313534203135363430332E383036343832393230352C203131393935352E3638323236323638343034203135363430342E30323931323634313635342C203131393934392E3932373739313837363434203135363430332E38383534393438323232332C203131393933362E3334333130323734313234203135363430312E38303430373838323334322C203131393933332E3136373530343937343534203135363430312E30373933333736383533342C203131393933332E3136313636393935323032203135363430312E30373830303936373832382C203131393932352E3339383931353130343236203135363339392E333136313530363536372C203131393931312E3432333939343233373738203135363339362E30333431393933363834342C203131393839372E3039383236363230343231203135363339322E333833373038303234342C203131393838372E3536303836303530343637203135363338392E35333937303138343934362C203131393838322E3834383433363831383431203135363338382E303032363738323934372C203131393837392E3438393430333536383132203135363338362E38383230303532373731382C203131393837392E3436333636313533363331203135363338362E38373334393434363434342C203131393837352E3931393631313339393936203135363338352E373132343132363737352C203131393837302E3037393131393930323437203135363338332E34343431353630393135342C203131393836392E3937383436373038323536203135363338332E343036333036333230352C203131393836302E3138333935313837323734203135363337392E38343238393030383834352C203131393834382E3131363031313238313633203135363337342E39313233373334343336332C203131393832332E3438393836343837343931203135363336342E35373735343437313832322C203131393832332E33383936323335383839203135363336342E353336373436383035382C203131393831372E31373039353132393431203135363336322E30383338373630363137382C203131393831362E3032333635373736383337203135363336312E363331313534383338392C203131393739362E3736333033363038313438203135363335342E30323733303431383939382C203131393739322E3932313639323739393935203135363335322E35313130363030333636332C203131393738342E3636373431343032393537203135363334382E353235333137363336332C203131393737392E3232323837353736313638203135363334332E39323636313635313436382C203131393737382E37313931363636323736203135363334332E32383330353633323735382C203131393737382E3034323239323836363738203135363334322E35373431323930343531382C203131393737372E3234303132303139333636203135363334322E303130383735313231372C203131393737362E33343334373536353432203135363334312E36313439343030383935322C203131393737352E3338363831363739353233203135363334312E34303135333935303938382C203131393737342E3430363930373438303138203135363334312E33373838373432343734332C203131393737332E34343134303530373435203135363334312E35343738313533313534352C203131393737322E3532373431333239343137203135363334312E39303138373034303334342C203131393737312E3730303035363333303735203135363334322E34323734333333373239342C203131393737302E3939313132393034383334203135363334332E313034333037313333382C203131393737302E3432373837353132343835203135363334332E393036343739383036392C203131393737302E3033313934303039323637203135363334342E38303331323433343633362C203131393736392E3831383533393531333036203135363334352E37353937383332303533342C203131393736392E3739353837343235303539203135363334362E37333936393235323033382C203131393736392E39363438313533313836203135363334372E37303531393439323630362C203131393737302E33313838373034303636203135363334382E36313931383637303633382C203131393737302E3834343433333337363131203135363334392E34343635343336363938322C203131393737312E3636353633333337343337203135363335302E34393537343336373139332C203131393737322E3337363634393131323331203135363335312E323333373732323338362C203131393737382E3639373634393131343633203135363335362E35373237373232333634322C203131393737392E3734393834383530313531203135363335372E32353535363230373439352C203131393738382E3733393834383439363134203135363336312E35393635363230373638332C203131393738392E3037383234363438363734203135363336312E37343438303733353530382C203131393739332E3039313130353135343038203135363336332E33323837353135363539352C203131393831322E33353139363339323036203135363337302E39333236393538313039322C203131393831332E3530303732303830373437203135363337312E33383539393435343632332C203131393831392E3637303033343938363234203135363337332E38313933393637303538332C203131393834342E32363831333531323634203135363338342E313432343535323834332C203131393834342E3331313932383237323737203135363338342E31363035393032353330342C203131393835362E3439313932383237323437203135363338392E31333638393032353034362C203131393835362E3637333533323932203135363338392E32303639393336373838382C203131393836362E3530393030323939313134203135363339322E3738353331303030372C203131393837322E3432343838303039383139203135363339352E303832383433393037332C203131393837322E3637383333383436303839203135363339352E31373335303535333339352C203131393837362E3333373435353930333531203135363339362E33373232383530323530352C203131393837392E3639393539363432373437203135363339372E34393339393437323131322C203131393837392E3733313536353432393338203135363339372E35303435343130363235322C203131393838342E3532303536353433333931203135363339392E30363635343130363532332C203131393838342E3634323139373334323031203135363339392E31303435303532393234362C203131393839342E33333731393733343233203135363430312E39393535303532393133362C203131393839342E3533313335313430343334203135363430322E30343931363639353636352C203131393930392E3030303335313430313132203135363430352E37333631363639353536342C203131393930392E3039313837313233353735203135363430352E37353835373139343238362C203131393932332E3133303837313233323833203135363430392E30353535373139343439382C203131393932332E3136373333303034343136203135363430392E30363339393033323236342C203131393933302E3934353431323132353531203135363431302E383239333238303630392C203131393933342E3239343439353032323936203135363431312E35393336363233313530382C203131393933342E36343937343830303936203135363431312E36363133323432393434332C203131393934382E3732373734383031313231203135363431332E38313833323432393232322C203131393934392E3336303233383835333937203135363431332E383734343433323231362C203131393935352E3631303233383835333937203135363431342E30333034343332323438322C203131393935352E39363531303239333835203135363431342E30323637303234353838352C203131393936312E3036363130323934323032203135363431332E37393137303234353537322C203131393936312E3234373538393533373537203135363431332E37383030333036303738362C203131393936372E3438323538393533363937203135363431332E3236353033303631312C203131393936372E3733343035373238363937203135363431332E323337383430343939312C203131393937312E33353532393431323632203135363431322E37353333343339383039332C203131393937342E3432313533303231333733203135363431322E333437383431373439372C203131393937352E30343532393639343031203135363431322E32323435373030343033362C203131393937362E3634373239363933383139203135363431312E38303035373030343136342C203131393938312E3833343433373138383337203135363431302E34323735333239313936322C203131393938322E3131303137313433333335203135363431302E33343539393334353731362C203131393938382E3634393137313433303433203135363430382E32303539393334353635362C203131393938382E3737383836323038313534203135363430382E31363135373235393638342C203131393939312E3930303538353531303634203135363430372E303434323933323930372C203131393939342E3430303636373432373132203135363430362E32323831333837373433342C203131393939342E3636333533303735393332203135363430362E31333431323834373338322C203131393939372E3134343933343035323337203135363430352E31363737323832393139372C203131393939372E38363437343430353935203135363430342E39303837343431353237352C203131393939382E3235343532343631313432203135363430342E37343936363733303336342C203131393939392E3932363532343630393831203135363430332E39383336363733303437342C203131393939392E3933353434303736303431203135363430332E39373935373139323430352C203132303030302E3735383434303736363739203135363430332E363030353731393233342C203132303030332E3439323631353039363331203135363430322E333431323535313339342C203132303030372E3833353830303433373334203135363430302E33353439383232323932342C203132303032362E3835363436353733333039203135363339322E323233343933323738362C203132303032362E3938363130313532313231203135363339322E31363538383433313639362C203132303034322E3133383638343335383137203135363338352E31373331353132323432372C203132303035322E3231373938373838313834203135363338302E37393034393933363235372C203132303035372E3835313431313631313437203135363337382E33343930393530363533382C203132303037312E3439353433303438353139203135363337342E33303734333532303833382C203132303038332E3934343838323930393638203135363337302E383533393333353231382C203132303130312E37303131363030343335203135363336362E343932353638393836332C203132303130352E3538343535303334343035203135363336352E373232353032363038332C203132303131312E3938383337383434303032203135363336342E34353135333637393432382C203132303134332E3237343030333634343632203135363335382E32353533343438303539322C203132303136372E38373135363735383139203135363335332E37333038363239323332362C203132303137332E3235303239393935313534203135363335322E373435393237343936332C203132303138372E3631323136313133393439203135363335302E353938303236373530382C203132303138392E3235393331333436303737203135363335302E34303631383937333535342C203132303139392E32303036323939383137203135363334392E33303936313932313238342C203132303231332E3832333534333532353936203135363334382E32373534393632353034382C203132303233312E3532333235353437303931203135363334372E36323239333131313136332C203132303235322E3235393837373330373131203135363334372E31353731363332353835352C203132303236382E3135383231343536393932203135363334372E32303333393037393034372C203132303238302E3539393539303135363032203135363334382E323437393538383038382C203132303238332E3539313938333036393936203135363334382E35303635363036363739372C203132303239322E3030313235333435363238203135363334392E35303435393838353839342C203132303239392E33353133313430343833203135363335302E39373934373533343436382C203132303330312E3437393735313238303935203135363335312E34313436393434343931372C203132303330392E3636373334323530323931203135363335332E393035383732383337332C203132303331352E3533363437333035343332203135363335362E34333835393337373633372C203132303331392E32383132313434383536203135363335382E303535343436353533342C203132303332372E3638333432363536323336203135363336322E353339363533383038382C203132303332382E3034333137343337323333203135363336322E37353536353632333137382C203132303332382E3035393834343732343132203135363336322E37363536323135363838362C203132303333362E3037373536333538393132203135363336372E353337343036303431372C203132303334332E34323832333936363037203135363337332E33323132333733333636342C203132303334352E3436393233343530323734203135363337352E33373132383331353834392C203132303334392E3538303739313538313637203135363337392E35303330333236333434372C203132303335362E3137373439323030393334203135363338372E34393537353839343939382C203132303336312E3735303339313638363531203135363339352E37363330303939353339352C203132303336372E3036353938393831343734203135363430372E323036323938313432322C203132303337342E3539323738333339363032203135363432352E353732323239303033332C203132303337352E3935353330373438383638203135363432392E32323937383832383030332C203132303338382E3530363933313435383637203135363436322E393531393131323731372C203132303339362E3639323930313233303731203135363438352E37353139373434383337352C203132303430312E3439363638383539343234203135363439392E31343233383137353038362C203132303430312E3531333139383836393734203135363439392E31383737313433313939372C203132303430332E3031353030363739353432203135363530332E32353031393437353736322C203132303430352E3331373831343831393232203135363530392E34383336373530383534372C203132303430382E3932323637343738323038203135363531392E32323031393737343435352C203132303431332E3234383335303534393831203135363533312E30383733333435353239382C203132303432312E3535303335333436353436203135363535332E38363030323336353737362C203132303432382E3331333932343538323135203135363537342E33323838323938363135372C203132303432392E3833323530323332333239203135363538302E31303037323739373939372C203132303433302E3739343037353136303635203135363538362E38373631313831333335342C203132303433312E3037323433393334333134203135363539352E30303232383332333334352C203132303433302E3433383731313734323139203135363630342E37353434333437313135362C203132303433302E3331363939363932383837203135363630352E35343731343134363739382C203132303432392E3532313335373234393832203135363631302E36333234343730333039322C203132303432382E3730313631383532333338203135363631352E38343837383434333534382C203132303432382E36393833333233393036203135363631352E38363939393231333434352C203132303432382E3631343335383138303031203135363631362E343139373330323632392C203132303432382E3630303634333630313937203135363631362E35303633393437323636332C203132303432352E3132323239353239363538203135363633302E393234383230323032352C203132303432312E3936303531333731323835203135363634322E31373430393537393033372C203132303432302E3337343337373337383538203135363634372E38313935383039363234342C203132303432302E3238323335313830383837203135363634382E32303532343436393538362C203132303431382E3439333335313830343334203135363635372E32383332343436393734362C203132303431382E3439323332353630323038203135363635372E32383834363636343738352C203132303431372E3431343332353630373933203135363636322E37383934363636343631352C203132303431372E3337353335383034303932203135363636332E30313537323735333838322C203132303431362E3139353335383034313232203135363637302E39353237323735343135332C203132303431362E3134383636393637393536203135363637312E34313131363434303831342C203132303431352E3939393636393637353633203135363637342E30393831363434303731332C203132303431352E3939383935303931323733203135363637342E3131313434353738332C203132303431352E3732353935303931303833203135363637392E323833343435373831342C203132303431352E3732333133383638323536203135363637392E37353033393534393733372C203132303431362E3033363133383638333537203135363638372E34333833393534393833382C203132303431362E30373936383638313633203135363638372E39323339303739343132372C203132303431372E3131383638363832303833203135363639352E33393239303739343137372C203132303431372E3230383030393037303433203135363639352E38363634363235353638352C203132303431382E3932363030393036353138203135363730332E30353334363235353538342C203132303431392E3036313033373337323336203135363730332E35313737363635353232382C203132303432312E3432313033373337313737203135363731302E33373637363635353333382C203132303432312E3631303434313835323734203135363731302E38343739373238323533342C203132303432342E3538363434313835363236203135363731372E32383539373238323633362C203132303432342E3638393632383532393833203135363731372E34393631333334323930332C203132303432352E3639373632383533313134203135363731392E3433333133333432382C203132303432352E3730323035323137353838203135363731392E343431363134323033362C203132303433362E3738383035323137313335203135363734302E363435363134323033352C203132303435382E3230373438363835393839203135363738312E353833343435353132392C203132303435382E3938333230373132353933203135363738332E30363639313035353738332C203132303435392E3832393236393632353238203135363738342E363834303330323733362C203132303437302E3636323832323930383437203135363830352E34313031373536343431322C203132303437342E3634383331383335343034203135363831332E30323330333433383937342C203132303437342E36353833323034353036203135363831332E30343230343033343733382C203132303437382E3830373332303434373039203135363832302E38383530343033343538362C203132303530302E3731333338323937353738203135363836322E32393231353835333736362C203132303530362E3531303534393633383833203135363837332E323435343733353938322C203132303531372E3632313537363131313439203135363839342E323439313831353531342C203132303532312E3032393531333937313233203135363930302E37363835383436333936382C203132303533372E3330333133363339383535203135363933312E393234383631373731342C203132303536382E3232353039323232383233203135363939312E313131373737323738392C203132303537342E3239343832333739343635203135373030322E37333632363331383630332C203132303538382E3335303233303533333935203135373032392E36343330343139393234342C203132303539392E3736373730373639383839203135373035312E34393433383630303933322C203132303630352E3832343938313131303432203135373036332E323034323438363136352C203132303631392E3931393833303633353139203135373039302E343233383930313837372C203132303635382E3130363435393337303438203135373136342E31383931373330313438332C203132303638352E3237303231363532373637203135373231362E36383837303336363235342C203132303638352E3237333638383232313332203135373231362E36393534303130333838332C203132303638352E3730333638383232313032203135373231372E35323334303130343034342C203132303638352E38313436323034393831203135373231372E37323534373936373938322C203132303639332E31313936323034393738203135373233302E33333434373936383039332C203132303639332E3336313537323731373937203135373233302E37313139393936313330342C203132303730322E3637333537323731363936203135373234332E38393939393936313033322C203132303730322E3933323536343939323732203135373234342E32333536333436303437332C203132303731342E3531383536343938383139203135373235382E30303136333436303733352C203132303731342E3738343238333337323439203135373235382E32393331383436333834362C203132303731362E3434383238333337373032203135373235392E39383031383436333734352C203132303731362E3435363031373036393732203135373235392E39383830303739393333352C203132303732352E3532373031373037323035203135373236392E31343430303739393238342C203132303732382E3938303139363939343935203135373237322E36323931383935393737332C203132303732392E3131363137323837353037203135373237322E37363133313832363337372C203132303734372E3238373933383438313935203135373238392E37363130393839383233332C203132303739382E3539383335303034383333203135373333372E37373436313236313334332C203132303830332E3433323939373037303431203135373334322E3330353334343639322C203132303833302E3331383637393338353337203135373336372E34363338353635333931372C203132303833302E33323530303236343638203135373336372E34363937363333333135322C203132303833352E3035333933333236353237203135373337312E38373935373538393835382C203132303834372E3632363935313333343738203135373338332E37373332393533353632382C203132303930382E3534313836313737373732203135373434312E33363831353633393636372C203132303934382E3838383635353132303632203135373437392E34393636323130393630342C203132303938312E3638303130313333383938203135373531302E393336363235393131342C203132313030382E3738313538323539303235203135373533362E39333631323832353734352C203132313031352E39313733343138333239203135373534332E37373838353634373731382C203132313031372E3332313830333232363832203135373534352E31323532393838363531372C203132313031382E3434393235323539303137203135373534362E32303537323936333138322C203132313034322E3234333936323535373232203135373536392E303135343531363031382C203132313034322E33393136333934373038203135373536392E313531343332393131382C203132313035362E3336373633393436363837203135373538312E353131343332393131322C203132313035362E3637353931383938323933203135373538312E37363239333539383039352C203132313037322E3439363931383938353236203135373539332E36353339333539383335372C203132313037322E3831303236323539313435203135373539332E38373132353334353730342C203132313039302E3535323236323539303134203135373630352E31393932353334353439322C203132313039302E3731313333373831353332203135373630352E32393636393138343538372C203132313131342E35303033333738313234203135373631392E32363436393138343830382C203132313131342E3639313935323739313234203135373631392E33373136313733323733332C203132313132322E3336333935323739373038203135373632332E34333436313733323436322C203132313132322E3337303036383933353735203135373632332E34333738353039343238342C203132313133382E33383530363839323839203135373633312E38393038353039343434342C203132313133382E38333935333939333236203135373633322E31303233313733373232322C203132313135372E3539373533393933333931203135373633392E37313133313733373333322C203132313135372E383637393437383832203135373633392E38313230323036323736342C203132313138322E3837353330323139363538203135373634382E33313138303131363433382C203132313230322E3330343635363532313938203135373635342E39323135383135303832342C203132313230322E3331363033353536303038203135373635342E393235343337333539322C203132313231362E3435303732393731383436203135373635392E363936313333333239382C203132313231382E3333363736363338303039203135373636302E33373538343736373637372C203132313231382E3432313036383732373435203135373636302E34303533383135303032362C203132313235382E3232313334393634333331203135373637332E393530373735343337312C203132313235392E3934333332363430333636203135373637342E35343036353637343833322C203132313331392E3733313938373036393239203135373639352E32323835313931333830342C203132313333302E3139383232313930393531203135373639392E343837343339373638352C203132313334302E3438373733373534373131203135373730352E31383935343234333735382C203132313334392E3239363332323935353337203135373731322E343734393332353732322C203132313334392E3433363833313637323632203135373731322E35393039323538333139362C203132313335362E3236303238373233303136203135373731382E323936303837333935352C203132313336322E3337363230343738373136203135373732342E313335393432353039332C203132313336372E3234353538363738343433203135373733302E31363535323138393238362C203132313337312E3131363530353832323131203135373733372E333931393330393831372C203132313337312E3636313738393337323337203135373733382E32303634323538313236382C203132313337322E3335353439353531303937203135373733382E38393838393038313034342C203132313337332E3137303936353530303033203135373733392E34343237313439333334352C203132313337342E3037363836313238353138203135373733392E38313639393933323536372C203132313337352E3033383336393739393435203135373734302E30303733363034343738342C203132313337362E3031383534303830393836203135373734302E30303634383238323837332C203132313337362E3937393730363839343134203135373733392E38313434303031393437372C203132313337372E3838343933303937383732203135373733392E34333834393431373338372C203132313337382E3639393432353830393637203135373733382E38393332313036323336322C203132313337392E3339313839303830373431203135373733382E3139393530343438352C203132313337392E3933353731343933303434203135373733372E33383430333434393539352C203132313338302E3330393939393332323636203135373733362E34373831333837313037382C203132313338302E3530303336303434343831203135373733352E353136363330313936352C203132313338302E34393934383238323537203135373733342E353336343539313836312C203132313338302E3330373430303139313736203135373733332E35373532393331303138342C203132313337392E3933313439343137303836203135373733322E36373030363930313732372C203132313337352E38333834393431373631203135373732352E303239303639303138333829291A2F687474703A2F2F7777772E6F70656E6769732E6E65742F6F6E742F67656F73706172716C23776B744C69746572616C86011283010A2412220A20396366333433396266356631316239656532623630633534626237343564396212330A310A2F687474703A2F2F7777772E77332E6F72672F313939392F30322F32322D7264662D73796E7461782D6E7323747970651A260A240A22687474703A2F2F7777772E77332E6F72672F6E732F6C6F636E2347656F6D657472798B011288010A2412220A20363963333161383563356231343566333834303731613233393066373637333412310A2F0A2D687474703A2F2F7777772E77332E6F72672F323030342F30322F736B6F732F636F726523707265664C6162656C1A2D1A2B0A2242657065726B746520646F6F7267616E6720766F6F7220766F657467616E6765727312056E6C2D6265', '2024-06-27 08:47:04.004000', '9f3a0219-e12d-4891-859f-6c9c5f967d48', diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/MemberPostgresRepository.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/MemberPostgresRepository.java index af3e9f00c..db56288dc 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/MemberPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/MemberPostgresRepository.java @@ -38,7 +38,7 @@ public List insertAll(List members) { final int collectionId = getCollectionId(members.getFirst().getCollectionName()); final List subjects = members.stream().map(IngestedMember::getSubject).toList(); if (!membersContainDuplicateIds(members) && !membersExistInCollection(collectionId, subjects)) { - String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model, old_id) VALUES (?,?,?,?,?,?,?)"; + String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model) VALUES (?,?,?,?,?,?)"; final List batchArgs = members.stream() .map(member -> new Object[]{ @@ -48,7 +48,6 @@ public List insertAll(List members) { member.getTimestamp(), member.getTransactionId(), modelConverter.convertToDatabaseColumn(member.getModel()), - member.getCollectionName() + "/" + member.getSubject() }) .toList(); @@ -71,12 +70,6 @@ protected boolean membersContainDuplicateIds(List members) { .size() != members.size(); } - @Override - public Stream findAllByIds(List memberIds) { - return repository.findAllByOldIdIn(memberIds) - .stream() - .map(mapper::toMember); - } @Override public Stream findAllByCollectionAndSubject(String collectionName, List subjects) { @@ -91,12 +84,6 @@ public void deleteMembersByCollectionNameAndSubjects(String collectionName, List repository.deleteAllByCollectionNameAndSubjectIn(collectionName, subjects); } - @Override - @Transactional - public void removeFromEventSource(List ids) { - jdbcTemplate.update("UPDATE members SET is_in_event_source = false WHERE member_id IN ?", ids); - } - @Override public Stream findAllByTreeNodeUrl(String url) { final String sql = """ diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java index 038962c80..64fc19b81 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/entity/MemberEntity.java @@ -12,7 +12,6 @@ @Entity @Table(name = "members", indexes = { @Index(columnList = "member_id, timestamp"), - @Index(columnList = "old_id"), @Index(columnList = "subject, collection_id", unique = true) }) public class MemberEntity { @@ -20,8 +19,6 @@ public class MemberEntity { @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "member_id", columnDefinition = "BIGINT", nullable = false) private long id; - @Column(name = "old_id", nullable = false) - private String oldId; @Column(name = "subject", nullable = false) private String subject; @ManyToOne(fetch = FetchType.EAGER) @@ -61,9 +58,7 @@ protected MemberEntity() { public long getId() { return id; } - public String getOldId() { - return oldId; - } + public String getSubject() { return subject; diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java index e2483ed55..6c6f8dfc7 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java @@ -28,8 +28,6 @@ select count(*) nativeQuery = true) boolean existsByCollectionAndSubjectIn(int collectionId, List subjects); - List findAllByOldIdIn(List oldIds); - List findAllByCollectionNameAndSubjectIn(String collectionName, List subjects); void deleteAllByCollectionNameAndSubjectIn(String collectionName, List subjects); diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberRepositorySteps.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberRepositorySteps.java index 2a2435454..24ccb1d20 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberRepositorySteps.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/MemberRepositorySteps.java @@ -1,9 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.entities.IngestedMember; -import io.cucumber.datatable.DataTable; import io.cucumber.java.DataTableType; -import io.cucumber.java.Transpose; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -77,11 +75,6 @@ public List memberIdTransformer(Map row) { return row.values().stream().toList(); } - @When("I try to retrieve the following members by Id") - public void iTryToRetrieveTheFollowingMembersByIdMemberIds(@Transpose DataTable table) { - members = memberRepository.findAllByIds(table.column(0).stream().toList()).toList(); - } - @Then("I expect a list of {int} members") public void iExpectAListOfMembers(int memberCount) { assertEquals(memberCount, members.size()); diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java index 3e6091e85..8dc8e9994 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java @@ -149,16 +149,16 @@ private void insertMembers(int count) { .mapToObj(this::mapToInsertAgs) .toList(); - jdbcTemplate.batchUpdate("INSERT INTO members VALUES (?, ?, ?, 1, true, '', ?, ?, 'http://test-data/mobility-hindrance/1')", batchArgs); + jdbcTemplate.batchUpdate("INSERT INTO members (member_id, subject, timestamp, collection_id, member_model, transaction_id, version_of) " + + "VALUES (?, ?, ?, 1, '', ?, 'http://test-data/mobility-hindrance/1')", batchArgs); } private Object[] mapToInsertAgs(int i) { final LocalDateTime timestamp = START_TIME.plusHours(i); final String subject = SUBJECT_TEMPLATE + (i + 1); - final String oldId = COLLECTION_NAME + "/" + subject; return new Object[]{ - i + 1, subject, oldId, timestamp, UUID.randomUUID() + i + 1, subject, timestamp, UUID.randomUUID() }; } diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-member-id.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-member-id.xml new file mode 100644 index 000000000..e4faff2b9 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-member-id.xml @@ -0,0 +1,13 @@ + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-tables-with-cascade.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-tables-with-cascade.xml new file mode 100644 index 000000000..00d2adf10 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/drop-old-tables-with-cascade.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/master.xml new file mode 100644 index 000000000..3a3b9eb81 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_5_0/master.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index 3061e20c4..5d7680456 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -8,5 +8,6 @@ + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java index 395bb21d3..ed78933c2 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/PaginationSteps.java @@ -89,7 +89,7 @@ public void theOldPageHasAGenericRelationToTheNewPage() { private void saveMembers(List memberIds) { var collectionId = eventStreamEntityRepository.findAll().getFirst().getId(); String eventStream = "http://example.com/es"; - String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model, old_id) VALUES (?,?,?,?,?,?,?)"; + String sql = "INSERT INTO members (subject, collection_id, version_of, timestamp, transaction_id, member_model) VALUES (?,?,?,?,?,?)"; final List batchArgs = memberIds.stream() diff --git a/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/MemberPropertiesPostgresRepository.java b/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/MemberPropertiesPostgresRepository.java index b3f5f922d..abedd8104 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/MemberPropertiesPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/MemberPropertiesPostgresRepository.java @@ -34,14 +34,6 @@ public MemberPropertiesPostgresRepository(EntityManager entityManager, this.propertiesMapper = propertiesMapper; this.memberEntityRepository = memberEntityRepository; } - @Override - public void removePageMemberEntity(Long id, String collectionName, String viewName) { - Query query = entityManager.createQuery("DELETE FROM PageMemberEntity p WHERE p.member.id = :memberId AND p.bucket.view.name = :viewName AND p.bucket.view.eventStream.name = :collectionName"); - query.setParameter("viewName", viewName); - query.setParameter("collectionName", collectionName); - query.setParameter("memberId", id); - query.executeUpdate(); - } @Override @Transactional diff --git a/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/repository/RetentionMemberEntityRepository.java b/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/repository/RetentionMemberEntityRepository.java index 972b3d34f..812671f24 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/repository/RetentionMemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-retention-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/postgres/repository/RetentionMemberEntityRepository.java @@ -11,7 +11,7 @@ public interface RetentionMemberEntityRepository extends JpaRepository { - void deleteAllByIdIn(List oldIds); + void deleteAllByIdIn(List ids); @Query("SELECT m FROM MemberEntity m JOIN ViewEntity v ON m.collection = v.eventStream JOIN PageMemberEntity pm ON pm.member = m WHERE v.name = :viewName AND v.eventStream.name = :collectionName AND CAST(m.timestamp as timestamp) < CAST(:timestamp as timestamp)") Stream findAllByViewNameAndTimestampBefore(String viewName, String collectionName, LocalDateTime timestamp); diff --git a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java index ae96f84df..12a89973c 100644 --- a/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java +++ b/ldes-server-maintenance/ldes-server-retention/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/retention/repositories/MemberPropertiesRepository.java @@ -11,8 +11,6 @@ public interface MemberPropertiesRepository { - void removePageMemberEntity(Long id, String collectionName, String viewName); - void deleteAllByIds(List id); void removeFromEventSource(List id); diff --git a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/repositories/MemberRepository.java b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/repositories/MemberRepository.java index e5b214666..b166be395 100644 --- a/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/repositories/MemberRepository.java +++ b/ldes-server-port-ingest/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/repositories/MemberRepository.java @@ -8,11 +8,7 @@ public interface MemberRepository { List insertAll(List members); - Stream findAllByIds(List memberIds); + Stream findAllByCollectionAndSubject(String collectionName, List subjects); - Stream findAllByCollectionAndSubject(String collectionName, List subject); - - void deleteMembersByCollectionNameAndSubjects(String collectionName, List oldIds); - - void removeFromEventSource(List ids); + void deleteMembersByCollectionNameAndSubjects(String collectionName, List subjects); } From 844edb1c7c616f2fc39ff75c07d47a9c4087423f Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:36:11 +0200 Subject: [PATCH 10/37] feat: charset included in headers (#1380) * feat: charset included in headers * fix: broken tests * fix: broken tests * chore: cleaner approach of adding charset to the headers * fix: broken export * chore: PR remarks --- .../AdminEventStreamsRestControllerTest.java | 33 +++++++---- .../AdminShapeRestControllerTest.java | 17 ++++-- .../AdminViewsRestControllerTest.java | 13 +++-- .../domain/encodig/CharsetEncodingConfig.java | 16 ++++++ .../server/domain/encodig/CharsetFilter.java | 14 +++++ .../src/main/java/module-info.java | 1 + .../ResponseToModelConverter.java | 3 +- .../rest/treenode/TreeNodeControllerTest.java | 56 +++++++++++++++++-- .../src/test/resources/member-utf8-charset.nq | 11 ++++ 9 files changed, 135 insertions(+), 29 deletions(-) create mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetEncodingConfig.java create mode 100644 ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetFilter.java create mode 100644 ldes-server-port-fetch-rest/src/test/resources/member-utf8-charset.nq diff --git a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java index 54c646daa..7a55af5d6 100644 --- a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java +++ b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminEventStreamsRestControllerTest.java @@ -10,6 +10,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.HttpModelConverter; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.PrefixAdderImpl; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.RdfModelConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig.CharsetEncodingConfig; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.FragmentationConfig; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; @@ -30,6 +31,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -51,7 +53,7 @@ @ContextConfiguration(classes = {AdminEventStreamsRestController.class, HttpModelConverter.class, EventStreamListHttpConverter.class, EventStreamHttpConverter.class, EventStreamConverterImpl.class, ViewSpecificationConverter.class, PrefixAdderImpl.class, ValidatorsConfig.class, - AdminRestResponseEntityExceptionHandler.class, RetentionModelExtractor.class, + AdminRestResponseEntityExceptionHandler.class, RetentionModelExtractor.class, CharsetEncodingConfig.class, FragmentationConfigExtractor.class, PrefixConstructor.class, RdfModelConverter.class}) class AdminEventStreamsRestControllerTest { private static final String COLLECTION = "name1"; @@ -113,7 +115,8 @@ void when_StreamsPresent_then_StreamsAreReturned() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams").accept(contentTypeNQuads)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeNQuads)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeNQuads)) .andExpect(IsIsomorphic.with(expectedEventStreamsModel)); verify(eventStreamService).retrieveAllEventStreams(); @@ -137,7 +140,8 @@ void when_StreamPresent_Then_StreamIsReturned() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/" + COLLECTION).accept(contentTypeNQuads)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeNQuads)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeNQuads)) .andExpect(IsIsomorphic.with(model)); verify(eventStreamService).retrieveEventStream(COLLECTION); @@ -149,7 +153,8 @@ void when_StreamNotPresent_Then_Returned404() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/" + COLLECTION).accept(contentTypeTurtle)) .andExpect(status().isNotFound()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)); + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)); verify(eventStreamService).retrieveEventStream(COLLECTION); } @@ -176,7 +181,8 @@ void when_eventStreamModelIsPut_then_eventStreamIsSaved_and_status200IsExpected( .content(readDataFromFile("eventstream/streams/ldes.ttl")) .contentType(contentTypeTurtle)) .andExpect(status().isCreated()) - .andExpect(content().contentType(contentTypeNQuads)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeNQuads)) .andExpect(IsIsomorphic.with(expectedModel)); verify(eventStreamService).createEventStream(any(EventStreamTO.class)); @@ -202,7 +208,8 @@ void when_eventStreamThatCreateVersionsModelIsPut_then_eventStreamIsSaved_and_st .content(readDataFromFile("eventstream/streams/ldes-create-versions.ttl")) .contentType(contentTypeTurtle)) .andExpect(status().isCreated()) - .andExpect(content().contentType(contentTypeNQuads)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeNQuads)) .andExpect(IsIsomorphic.with(expectedModel)); verify(eventStreamService).createEventStream(any(EventStreamTO.class)); @@ -214,7 +221,8 @@ void when_ModelWithoutType_Then_ReturnedBadRequest() throws Exception { .content(readDataFromFile("eventstream/streams/ldes-without-type.ttl")) .contentType(contentTypeTurtle)) .andExpect(status().isBadRequest()) - .andExpect(content().contentType(contentTypeTurtle)); + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeTurtle)); verifyNoInteractions(eventStreamService); } @@ -225,8 +233,8 @@ void when_MalformedModelInRequestBody_Then_ReturnedBadRequest() throws Exception .content(readDataFromFile("eventstream/streams/malformed-ldes.ttl")) .contentType(contentTypeTurtle)) .andExpect(status().isBadRequest()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)); - + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)); verifyNoInteractions(eventStreamService); } } @@ -247,8 +255,8 @@ void when_deleteNotExistingCollection_then_expectStatus404() throws Exception { mockMvc.perform(delete("/admin/api/v1/eventstreams/name1")) .andExpect(status().isNotFound()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)); - + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)); verify(eventStreamService).deleteEventStream(COLLECTION); } } @@ -269,7 +277,8 @@ void when_collectionDoesNotExist_then_expectStatus404() throws Exception { mockMvc.perform(post("/admin/api/v1/eventstreams/name1/close")) .andExpect(status().isNotFound()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)); + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)); verify(eventStreamService).closeEventStream(COLLECTION); } diff --git a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminShapeRestControllerTest.java b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminShapeRestControllerTest.java index d45e5630f..2b5228c4c 100644 --- a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminShapeRestControllerTest.java +++ b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminShapeRestControllerTest.java @@ -9,6 +9,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.HttpModelConverter; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.PrefixAdderImpl; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.RdfModelConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig.CharsetEncodingConfig; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFDataMgr; @@ -29,6 +30,7 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Objects; @@ -46,7 +48,7 @@ @ActiveProfiles({"test", "rest"}) @ContextConfiguration(classes = {AdminShapeRestController.class, HttpModelConverter.class, PrefixAdderImpl.class, AdminRestResponseEntityExceptionHandler.class, ValidatorsConfig.class, - RdfModelConverter.class}) + RdfModelConverter.class, CharsetEncodingConfig.class}) class AdminShapeRestControllerTest { @MockBean private ShaclShapeService shaclShapeService; @@ -76,7 +78,8 @@ void when_ShapeIsPresentArePresent_Then_ShapeIsReturned() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/" + collectionName + "/shape") .accept(contentTypeTurtle)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeTurtle)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeTurtle)) .andExpect(IsIsomorphic.with(expectedShapeModel)); } @@ -88,7 +91,8 @@ void when_ViewNotPresent_Then_Returned404() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/" + collectionName + "/shape") .accept(contentTypeTurtle)) .andExpect(status().isNotFound()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)) .andExpect(content().string("Resource of type: shacl-shape with id: %s could not be found.".formatted(collectionName))); } } @@ -106,7 +110,8 @@ void when_ModelInRequestBody_Then_MethodIsCalled() throws Exception { .content(readDataFromFile(fileName)) .contentType(contentTypeTurtle)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeTurtle)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeTurtle)) .andExpect(IsIsomorphic.with(expectedShapeModel)); InOrder inOrder = inOrder(shaclShapeValidator, shaclShapeService); @@ -124,8 +129,8 @@ void when_ModelWithoutType_Then_ReturnedBadRequest() throws Exception { .content(readDataFromFile("shacl/shape-without-type.ttl")) .contentType(contentTypeTurtle)) .andExpect(status().isBadRequest()) - .andExpect(content().contentType(contentTypeTurtle)); - + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeTurtle)); verify(shaclShapeValidator).validate(any()); } diff --git a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminViewsRestControllerTest.java b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminViewsRestControllerTest.java index 1b632cb81..e7e9dce7a 100644 --- a/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminViewsRestControllerTest.java +++ b/ldes-server-admin/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/admin/rest/controllers/AdminViewsRestControllerTest.java @@ -14,6 +14,7 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.HttpModelConverter; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.PrefixAdderImpl; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.RdfModelConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig.CharsetEncodingConfig; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewSpecification; @@ -35,6 +36,7 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; @@ -50,7 +52,7 @@ @WebMvcTest @ActiveProfiles({"test", "rest"}) -@ContextConfiguration(classes = {AdminViewsRestController.class, PrefixAdderImpl.class, +@ContextConfiguration(classes = {AdminViewsRestController.class, CharsetEncodingConfig.class, PrefixAdderImpl.class, HttpModelConverter.class, ViewHttpConverter.class, ListViewHttpConverter.class, ViewSpecificationConverter.class, ValidatorsConfig.class, AdminRestResponseEntityExceptionHandler.class, RetentionModelExtractor.class, @@ -78,7 +80,8 @@ void when_StreamAndViewsArePresent_Then_ViewsAreReturned() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/{collectionName}/views", COLLECTION_NAME) .accept(contentTypeNQuads)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeNQuads)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeNQuads)) .andExpect(IsIsomorphic.with(expectedViewModel1.add(expectedViewModel2))); } @@ -91,7 +94,8 @@ void when_StreamAndViewArePresent_Then_ViewIsReturned() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/{collectionName}/views/{viewName}", COLLECTION_NAME, VIEW_NAME) .accept(contentTypeTurtle)) .andExpect(status().isOk()) - .andExpect(content().contentType(contentTypeTurtle)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(contentTypeTurtle)) .andExpect(IsIsomorphic.with(expectedViewModel)); } @@ -103,7 +107,8 @@ void when_ViewNotPresent_Then_Returned404() throws Exception { mockMvc.perform(get("/admin/api/v1/eventstreams/{collectionName}/views/{viewName}", COLLECTION_NAME, VIEW_NAME) .accept(contentTypeTurtle)) .andExpect(status().isNotFound()) - .andExpect(content().contentType(MediaType.TEXT_PLAIN)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(MediaType.TEXT_PLAIN)) .andExpect(content().string("Resource of type: view with id: %s/%s could not be found.".formatted(COLLECTION_NAME, VIEW_NAME))); } diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetEncodingConfig.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetEncodingConfig.java new file mode 100644 index 000000000..6cae4bc21 --- /dev/null +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetEncodingConfig.java @@ -0,0 +1,16 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig; + +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class CharsetEncodingConfig { + @Bean + public FilterRegistrationBean registerCharsetFilter() { + FilterRegistrationBean filterRegistrationBean = new FilterRegistrationBean<>(); + filterRegistrationBean.setFilter(new CharsetFilter()); + filterRegistrationBean.addUrlPatterns("/*"); + return filterRegistrationBean; + } +} diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetFilter.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetFilter.java new file mode 100644 index 000000000..488be18c8 --- /dev/null +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/encodig/CharsetFilter.java @@ -0,0 +1,14 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig; + +import jakarta.servlet.*; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class CharsetFilter implements Filter { + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + response.setCharacterEncoding(StandardCharsets.UTF_8.name()); + chain.doFilter(request, response); + } +} diff --git a/ldes-server-domain/src/main/java/module-info.java b/ldes-server-domain/src/main/java/module-info.java index ec845fd5a..9ac61dfb0 100644 --- a/ldes-server-domain/src/main/java/module-info.java +++ b/ldes-server-domain/src/main/java/module-info.java @@ -5,6 +5,7 @@ exports be.vlaanderen.informatievlaanderen.ldes.server.domain.constants; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.model; exports be.vlaanderen.informatievlaanderen.ldes.server.domain.rest; + exports be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig; // Events exports be.vlaanderen.informatievlaanderen.ldes.server.domain.events.retention; diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/ResponseToModelConverter.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/ResponseToModelConverter.java index c60917e05..0da861b9c 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/ResponseToModelConverter.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/resultactionsextensions/ResponseToModelConverter.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.resultactionsextensions; +import org.apache.jena.atlas.web.ContentType; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.RDFLanguages; import org.apache.jena.riot.RDFParser; @@ -18,7 +19,7 @@ public ResponseToModelConverter(MockHttpServletResponse response) { public Model convert() throws UnsupportedEncodingException { return RDFParser .fromString(response.getContentAsString()) - .lang(RDFLanguages.contentTypeToLang(response.getContentType())) + .lang(RDFLanguages.contentTypeToLang(ContentType.create(response.getContentType()))) .toModel(); } } diff --git a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java index 4746c30e5..54a0d2121 100644 --- a/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java +++ b/ldes-server-port-fetch-rest/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/rest/treenode/TreeNodeControllerTest.java @@ -3,9 +3,11 @@ import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.PrefixAdder; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.PrefixAdderImpl; import be.vlaanderen.informatievlaanderen.ldes.server.domain.converter.RdfModelConverter; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.encodig.CharsetEncodingConfig; import be.vlaanderen.informatievlaanderen.ldes.server.domain.events.admin.EventStreamCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.server.domain.exceptions.MissingResourceException; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.*; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.EventStream; +import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.rest.PrefixConstructor; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.Member; import be.vlaanderen.informatievlaanderen.ldes.server.fetching.entities.TreeNode; @@ -48,6 +50,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -59,6 +62,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.ServerConfig.HOST_NAME_KEY; import static org.apache.jena.riot.WebContent.contentTypeTurtle; import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @@ -70,7 +74,7 @@ RestConfig.class, TreeViewWebConfig.class, RestResponseEntityExceptionHandler.class, PrefixConstructor.class, RdfModelConverter.class, TreeNodeStreamConverterImpl.class, PrefixAdderImpl.class, - TreeNodeStatementCreatorImpl.class}) + TreeNodeStatementCreatorImpl.class, CharsetEncodingConfig.class}) class TreeNodeControllerTest { private static final String COLLECTION_NAME = "ldes-1"; private static final String FRAGMENTATION_VALUE_1 = "2020-12-28T09:36:09.72Z"; @@ -93,7 +97,6 @@ class TreeNodeControllerTest { private RestConfig restConfig; @Autowired private CachingStrategy cachingStrategy; - private WebTestClient client; @BeforeEach void setUp() { @@ -103,7 +106,7 @@ void setUp() { @ParameterizedTest(name = "Correct getting of an open LdesFragment from the REST Service with mediatype{0}") @ArgumentsSource(MediaTypeRdfFormatsArgumentsProvider.class) void when_GETRequestIsPerformed_ResponseContainsAnLDesFragment(String mediaType, Lang lang, boolean immutable, - String expectedHeaderValue, String expectedEtag) throws Exception { + String expectedHeaderValue, String expectedEtag) throws Exception { EventStream eventStream = new EventStream(COLLECTION_NAME, null, null, false); eventPublisher.publishEvent(new EventStreamCreatedEvent(eventStream)); @@ -129,7 +132,8 @@ void when_GETRequestIsPerformed_ResponseContainsAnLDesFragment(String mediaType, .andExpect(status().isOk()) .andExpect(header().string("Cache-Control", expectedHeaderValue)) .andExpect(header().string("Etag", "\"" + expectedEtag + "\"")) - .andExpect(content().contentType(expectedContentType)) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(expectedContentType)) .andReturn(); Optional maxAge = extractMaxAge(result.getResponse().getHeader("Cache-Control")); @@ -141,6 +145,39 @@ void when_GETRequestIsPerformed_ResponseContainsAnLDesFragment(String mediaType, verify(treeNodeFetcher, times(1)).getFragment(ldesFragmentRequest); } + + @Test + void given_MemberWithSpecialChars_when_GetRequestIsPerformed_then_CharsAreRightfullyEncoded() throws Exception { + EventStream eventStream = new EventStream(COLLECTION_NAME, null, null, false); + eventPublisher.publishEvent(new EventStreamCreatedEvent(eventStream)); + + LdesFragmentRequest ldesFragmentRequest = new LdesFragmentRequest(ViewName.fromString(fullViewName), + List.of(new FragmentPair(GENERATED_AT_TIME, FRAGMENTATION_VALUE_1))); + final String fragmentId = new LdesFragmentIdentifier(ldesFragmentRequest.viewName(), + ldesFragmentRequest.fragmentPairs()) + .asDecodedFragmentId(); + final Member member = createMember(); + TreeNode treeNode = new TreeNode(fragmentId, true, false, List.of(), + List.of(member), COLLECTION_NAME, null); + + when(treeNodeFetcher.getFragment(ldesFragmentRequest)).thenReturn(treeNode); + + + String mediaType = "text/turtle"; + mockMvc + .perform(get("/{collectionName}/{viewName}", COLLECTION_NAME, VIEW_NAME) + .param("generatedAtTime", FRAGMENTATION_VALUE_1) + .accept(mediaType)) + .andExpect(status().isOk()) + .andExpect(content().encoding(StandardCharsets.UTF_8)) + .andExpect(content().contentTypeCompatibleWith(mediaType)) + .andExpect(content().string(containsString("ë"))) + .andExpect(content().string(containsString("你好"))) + .andReturn(); + + verify(treeNodeFetcher, times(1)).getFragment(ldesFragmentRequest); + } + private List getObjectURIs(Model model, Property property) { return model .listStatements(null, property, (Resource) null) @@ -202,6 +239,7 @@ void when_GETRequestIsPerformedOnOtherCollectionName_ResponseIs404() throws Exce static class MediaTypeRdfFormatsArgumentsProvider implements ArgumentsProvider { @Override + @SuppressWarnings("java:S1874") // deprecated Lang used for testing backwards compatibility public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of("application/n-quads", Lang.NQUADS, true, @@ -241,7 +279,7 @@ void when_GETRequestIsPerformedForStreaming_ResponseContainsAnLDesFragment() { when(streamingTreeNodeFactory.getMembersOfFragment(identifier)) .thenReturn(Stream.of(new Member("member1", ModelFactory.createDefaultModel()), new Member("member2", ModelFactory.createDefaultModel()))); - client = WebTestClient.bindToController(new TreeNodeController(restConfig, treeNodeFetcher, + WebTestClient client = WebTestClient.bindToController(new TreeNodeController(restConfig, treeNodeFetcher, streamingTreeNodeFactory, treeNodeStreamConverter, cachingStrategy)).build(); client.get() @@ -272,4 +310,10 @@ public CachingStrategy cachingStrategy(@Value(HOST_NAME_KEY) String hostName) { return new EtagCachingStrategy(hostName); } } + + private static Member createMember() { + final String subject = "http://data.lblod.info/id/public-service-snapshot/46969c9a-a803-ef11-9f8a-000d3aad8fb9"; + final Model model = RDFParser.source("member-utf8-charset.nq").toModel(); + return new Member(subject, model); + } } diff --git a/ldes-server-port-fetch-rest/src/test/resources/member-utf8-charset.nq b/ldes-server-port-fetch-rest/src/test/resources/member-utf8-charset.nq new file mode 100644 index 000000000..988f98bcc --- /dev/null +++ b/ldes-server-port-fetch-rest/src/test/resources/member-utf8-charset.nq @@ -0,0 +1,11 @@ + "2024-04-29T14:57:19Z"^^ . + . + . + _:Bac223c44717bb273efd762e7f709484c . + "Subsidie verfraaiing handelspanden aanvragen"@nl-be-x-informal . + . + . +_:Bac223c44717bb273efd762e7f709484c "Hoeveel bedraagt de subsidie?"@nl-be-x-informal . +_:Bac223c44717bb273efd762e7f709484c "

Mede dankzij de extra financiële ondersteuning van VLAIO (het Vlaams Agentschap voor Innoveren en Ondernemen)"@nl-be-x-informal . +_:Bac223c44717bb273efd762e7f709484c "你好"@nl-be-x-informal . +_:Bac223c44717bb273efd762e7f709484c . \ No newline at end of file From d1435e171c16e77da702d5d3f857d28fab466e78 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:27:46 +0200 Subject: [PATCH 11/37] fix: broken compaction rework (#1383) * fix: broken compaction rework * feat: make failing background tasks more visible in tests * chore: PR remarks * fix: unused imports --- .../ldes/server/LdesServerIntegrationTest.java | 3 +++ .../ldes/server/LdesServerSteps.java | 13 +++++++++++++ .../features/compaction/compaction.feature | 3 ++- .../resources/features/retention/retention.feature | 6 ++++++ .../services/CompactionCandidateSorter.java | 12 ++++++------ .../application/valueobjects/CompactedPages.java | 4 ++-- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java index 7bc57fbf0..9c74037d4 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java @@ -9,6 +9,7 @@ import io.cucumber.spring.CucumberContextConfiguration; import io.zonky.test.db.AutoConfigureEmbeddedDatabase; import jakarta.persistence.EntityManager; +import org.springframework.batch.core.explore.JobExplorer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability; @@ -63,4 +64,6 @@ public class LdesServerIntegrationTest { @Autowired DataSource dataSource; + @Autowired + JobExplorer jobExplorer; } diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java index bdc8c3969..8ef6946f9 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerSteps.java @@ -16,6 +16,9 @@ import org.apache.jena.riot.*; import org.apache.jena.vocabulary.RDF; import org.awaitility.Awaitility; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobInstance; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpServletResponse; @@ -44,6 +47,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE_MEMBER; import static be.vlaanderen.informatievlaanderen.ldes.server.domain.constants.RdfConstants.TREE_REMAINING_ITEMS; +import static be.vlaanderen.informatievlaanderen.ldes.server.maintenance.batch.MaintenanceFlows.MAINTENANCE_JOB; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.apache.jena.rdf.model.ResourceFactory.createResource; @@ -362,4 +366,13 @@ public void theResponseFromRequestingTheUrlDoesContainAJsonFile(String message, .andReturn().getResponse(); assertTrue(response.getContentAsString().contains("%s{collection=\"%s\"} %s".formatted(message, collection, value))); } + + @And("the background processes did not fail") + public void theBackgroundProcessesDidNotFail() { + JobInstance lastJobInstance = Objects.requireNonNull(jobExplorer.getLastJobInstance(MAINTENANCE_JOB)); + JobExecution lastJobExecution = Objects.requireNonNull(jobExplorer.getLastJobExecution(lastJobInstance)); + boolean hasFailedExecutions = lastJobExecution.getStepExecutions().stream() + .anyMatch(stepExecution -> stepExecution.getStatus().equals(BatchStatus.FAILED)); + assertThat(hasFailedExecutions).isFalse(); + } } diff --git a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature index c73719ec3..ee9c102a2 100644 --- a/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature +++ b/ldes-server-integration-test/src/test/resources/features/compaction/compaction.feature @@ -20,4 +20,5 @@ Feature: Execute CompactionService | 3 | And verify the following pages no longer exist | 2 | - | 3 | \ No newline at end of file + | 3 | + And the background processes did not fail \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/resources/features/retention/retention.feature b/ldes-server-integration-test/src/test/resources/features/retention/retention.feature index 3444ac21e..304468c3f 100644 --- a/ldes-server-integration-test/src/test/resources/features/retention/retention.feature +++ b/ldes-server-integration-test/src/test/resources/features/retention/retention.feature @@ -7,6 +7,7 @@ Feature: LDES Server Retention Then the first fragment of the "paged" view in collection contains members # Since all added members' timestamp values equal to their ingestion date, they should be removed after 15 seconds Then the first fragment of the "paged" view in collection contains 0 members + And the background processes did not fail Examples: | eventStreamDescriptionFile | template | collection | ingestedMemberCount | | "data/input/eventstreams/retention/mobility-hindrances_timebased.ttl" | "data/input/members/mob-hind.template.ttl" | "mobility-hindrances" | 30 | @@ -17,6 +18,7 @@ Feature: LDES Server Retention Given I create the eventstream When I ingest 30 members of template