Merge pull request #1282 from Informatievlaanderen/develop

ci: finalise 2.15.0
Informatievlaanderen · May 10, 2024 · 31fb33b · 31fb33b
2 parents 728589f + f64840c
commit 31fb33b
Show file tree

Hide file tree

Showing 71 changed files with 907 additions and 336 deletions.
diff --git a/docs/_configuration/openapi.json b/docs/_configuration/openapi.json
@@ -1,5 +1,10 @@
 {
   "openapi": "3.0.1",
+  "info": {
+    "title": "LDES Server Admin API",
+    "description": "This API makes it possible to manage an LDES Server",
+    "version": "2.15.0-SNAPSHOT"
+  },
   "servers": [
     {
       "url": "/",

diff --git a/docs/_features/compaction.md b/docs/_features/compaction.md
@@ -6,12 +6,13 @@ nav_order: 1
 
 # Fragment Compaction
 
-Compaction is a process that allows the server to merge immutable fragments that are underutilized (i.e. there are fewer members in the fragment than indicated in the `pageSize` of the view).
-Merging the fragments will result in a new fragment and the members and relations of the compacted fragments will be "copied" to the new fragment.
-This process runs entirely in the background. By default, the fragments that have been compacted will remain available for 7 days, `PD7`, but it can be configured differently. This to make sure that clients who are in the process of consuming the fragments have the time to continue consumption and get to the end of the Event Stream. When the period expires, the compacted fragments will be deleted.
+To prevent that clients need to read fragments which are almost empty, the system foresees an algorithm to merge such fragments together so that fragment size stays as optimal as possible.  
+This algorithm is called compaction and allows the server to merge immutable fragments that are underutilized (i.e. there are fewer members in the fragment than indicated in the [`tree:pageSize`](../_configuration/view) object of the view).
 
+Merging the fragments will result in a new fragment and the members and relations of the compacted fragments will be "copied" to the new fragment.  
+- Clients who haven't consumed the compacted fragments yet will be directed towards the new fragment, skipping the smaller, compacted fragments.
+- Clients who are in the process of consuming the compacted fragments will be able to continue consumption of the compacted fragments for a limited, configurable, amount of time.   The default value for conserving the compacted fragments is 7 days (`PD7`), unless configured otherwise.
 
-To configure this interval, please refer to the [Configuration Page.](../../how-to-run#ldes-server-config)
 
 ````mermaid
 %%{init: { 'gitGraph': {'mainBranchName': 'stream'}}}%%
@@ -29,3 +30,8 @@ gitGraph
     merge compaction tag: "compacted Stream" type: HIGHLIGHT
     commit id: "Fragment 6: 75% (Open)"
 ````
+
+
+Compaction wil run at configurable moments, so that the optimal time can be determined and impact on performance can be limited.  
+
+Configuration parameters of the compaction algorithm can be found on the [LDES Server Config](../how-to-run#ldes-server-config) section of the _How to run_ page.
diff --git a/docs/_fetch/http.md b/docs/_fetch/http.md
@@ -31,3 +31,20 @@ Depending on whether any fragmentations are defined, this either contains a part
 or one or multiple`tree:Relation` that point to partitioned fragment pages.
 
 These partitioned fragments contain the actual members.
+
+### Retrieving a fragment in a streaming way
+
+When retrieving a fragment, it is possible to receive the fragment page in a streaming way by specifying the accept type as `text/event-stream`.
+The fragment will then always be converted to the `application/rdf+protobuf` format.
+Because server-side events are used we encode the data to base64.
+
+When retrieving a fragment in this way, not all the data will be received at once.
+
+1. First the statements relating to the fragment itself and its relations to other fragments will be send. This event will have the name: `metadata`.
+2. Then all members will be send one by one. These events will have the name: `member`.
+
+Every piece of the fragment data is wrapped in the data part of a Server-side event.
+
+This method is useful when retrieving large fragments because you do not have to wait until every member is fetched and the entire fragment is constructed to start processing the data.
+When consuming a fragment in this way, it is important to recognise that an error can still occur after receiving an 200 OK response from the server.
+When such an error occurs, the error message will be send in a Server-side event named `error`.
diff --git a/docs/_includes/head_custom.html b/docs/_includes/head_custom.html
@@ -1,77 +1,81 @@
 <div class="test-header" data-v-df634f57="" data-v-7a7a37b1="">
-    <div class="global-header" data-v-df634f57=""><img src="{{ '/assets/images/LOGO_Vlaanderen_NIV2.svg' | relative_url }}"
+    <div class="global-header" data-v-df634f57=""><img
+            src="{{ '/assets/images/LOGO_Vlaanderen_NIV2.svg' | relative_url }}"
             alt="Vlaanderen" data-v-df634f57=""><img class="small-global-header-divider"
-            src="{{ '/assets/images/divider_globalheader.svg' | relative_url }}" alt="divider" data-v-df634f57=""><span
+                                                     src="{{ '/assets/images/divider_globalheader.svg' | relative_url }}"
+                                                     alt="divider" data-v-df634f57=""><span
             class="header-title header-font" data-v-df634f57="">Vlaamse Smart Data Space</span><span class="spacer"
-            data-v-df634f57=""></span>
-        <div id="help-needed" data-v-df634f57=""><span class="header-font" data-v-df634f57="">Hulp nodig</span><img
-                class="question-menu-item" src="{{ '/assets/images/question-mark.svg' | relative_url }}" alt="?"
-                data-v-df634f57=""></div>
+                                                                                                     data-v-df634f57=""></span>
+        <div id="help-needed" data-v-df634f57="">
+            <span class="header-font" data-v-df634f57="">Hulp nodig</span>
+            <a href="https://www.vlaanderen.be/vlaamse-smart-data-space-portaal/contact" target="_blank">
+                <img class="question-menu-item" src="{{ '/assets/images/question-mark.svg' | relative_url }}" alt="?"
+                     data-v-df634f57="">
+            </a>
+        </div>
     </div>
     <hr class="divider" data-v-df634f57="">
 </div>
 
 
-
 <style>
 
-#help-needed[data-v-df634f57] {
-    display: flex;
-    align-items: center;
-    gap: 5px;
-    background-image: url(/assets/images/hulp_nodig.svg | relative_url);
-    width: 144px;
-    height: 44px;
-}
-
-.global-header[data-v-df634f57] {
-    display: flex;
-    margin-left: 0px;
-    align-items: center;
-    position: fixed;
-    width: 100%;
-    border-bottom:0.5px solid rgb(138, 138, 138);
-    background-color: white;
-    z-index: 100;
-
-}
-
-.test-header{
-    margin-bottom: 20px;
-    overflow:hidden;
-}
-
-
-
-element.style {
-}
-
-.small-global-header-divider[data-v-df634f57] {
-    margin: 0 8px 0 4px;
-}
-
-.header-font {
-    color: #333332;
-    font-family: Flanders Art Sans, sans-serif;
-    font-size: 12px;
-    font-style: normal;
-    font-weight: 500;
-    line-height: normal;
-    letter-spacing: 0.5px;
-    text-transform: uppercase;
-}
-
-.spacer {
-    flex-grow: 1;
-}
-
-.divider {
-    border: 1px solid #8f8f8f66;
-    margin: 0;
-}
-
-#help-needed > span[data-v-df634f57] {
-    margin-left: 25px;
-}
+    #help-needed[data-v-df634f57] {
+        display: flex;
+        align-items: center;
+        gap: 5px;
+        background-image: url(/assets/images/hulp_nodig.svg | relative_url);
+        width: 144px;
+        height: 44px;
+    }
+
+    .global-header[data-v-df634f57] {
+        display: flex;
+        margin-left: 0px;
+        align-items: center;
+        position: fixed;
+        width: 100%;
+        border-bottom: 0.5px solid rgb(138, 138, 138);
+        background-color: white;
+        z-index: 100;
+
+    }
+
+    .test-header {
+        margin-bottom: 20px;
+        overflow: hidden;
+    }
+
+
+    element.style {
+    }
+
+    .small-global-header-divider[data-v-df634f57] {
+        margin: 0 8px 0 4px;
+    }
+
+    .header-font {
+        color: #333332;
+        font-family: Flanders Art Sans, sans-serif;
+        font-size: 12px;
+        font-style: normal;
+        font-weight: 500;
+        line-height: normal;
+        letter-spacing: 0.5px;
+        text-transform: uppercase;
+    }
+
+    .spacer {
+        flex-grow: 1;
+    }
+
+    .divider {
+        border: 1px solid #8f8f8f66;
+        margin: 0;
+    }
+
+    #help-needed > span[data-v-df634f57] {
+        margin-left: 25px;
+    }
 
 </style>
diff --git a/docs/_sass/_layout.scss b/docs/_sass/_layout.scss
@@ -118,4 +118,15 @@
 .highlight .n {
   color: #008000;
   font-weight: normal;
+}
+
+@media (min-width: 66.5rem) {
+  .side-bar {
+    width: calc((100% - 90rem) / 2 + 16.5rem);
+    min-width: 16.5rem;
+  }
+
+  .main {
+    margin-left: Max(16.5rem, calc((100% - 90rem) / 2 + 16.5rem));
+  }
 }
diff --git a/docs/_sass/theme.scss b/docs/_sass/theme.scss
@@ -215,6 +215,10 @@ a:link a:visited a:hover a:active {
 
 }
 
+.main {
+    max-width: 66.5rem!important;
+}
+
 .main-content {
     margin-top: 100px;
 }

diff --git a/docs/how-to-run.md b/docs/how-to-run.md
@@ -53,32 +53,25 @@ Here is an explanation provided for all the possibilities on how to tweak and co
   <tr><th>Property</th><th>Description</th><th>Required</th><th>Default value</th></tr>
 </thead>
 <tbody>
-  <tr><td colspan="4"><b>Open-API</b></td></tr>
-  <tr>
-    <td>springdoc.api-docs.enabled</td>
-    <td>This boolean indicates whether Open API documentation is enabld. <a href="https://springdoc.org/#properties">More Open-API documentation</a></td>
-    <td>No</td>
-    <td>true</td>
-  </tr>
+  <tr><td colspan="4"><b>API endpoints documentation</b></td></tr>
   <tr>
     <td>springdoc.api-docs.path</td>
-    <td>When enabled, an url* needs to be configured that points to the Open API documentation. <a href="https://springdoc.org/#properties">More Open-API documentation</a></td>
-    <td>No</td>
-  </tr>
-  <tr><td colspan="4"><b>Swagger UI</b></td></tr>
-  <tr>
-    <td>springdoc.swagger-ui.enabled</td>
-    <td>This boolean indicates whether Swagger API is enabled. This can be used to easily configure your Streams. <a href="https://springdoc.org/#swagger-ui-properties">More Swagger UI documentation</a></td>
+    <td>The url<sup>1</sup> that will point to the Open API documentation. <a href="https://springdoc.org/#properties">More Open-API documentation</a></td>
     <td>No</td>
-    <td></td>
   </tr>
   <tr>
     <td>springdoc.swagger-ui.path</td>
-    <td>When enabled, an url* needs to be configured that points to the Swagger documentation. <a href="https://springdoc.org/#swagger-ui-properties">More Swagger UI documentation</a></td>
+    <td>The url<sup>1</sup> that will point to the Swagger documentation. <a href="https://springdoc.org/#swagger-ui-properties">More Swagger UI documentation</a></td>
     <td>No</td>
     <td>true</td>
   </tr>
   <tr><td colspan="4"><b>URL Configuration</b></td></tr>
+  <tr>
+    <td>server.address</td>
+    <td>This is the url that will be used by the server application to bind to. This is especially useful when exposing actuator endpoints publicly. However, it must be known if the address cannot be found or is unavailable, the application will be unable to start.</td>
+    <td>No</td>
+    <td></td>
+  </tr>
   <tr>
     <td>ldes-server.host-name</td>
     <td>This is the url that will be used throughout the fragment names. This should therefor point to a resolvable url.</td>
@@ -104,7 +97,7 @@ Here is an explanation provided for all the possibilities on how to tweak and co
     <td>No</td>
     <td>604800</td>
   </tr>
-  <tr><td colspan="4"><b>MongoDB Storage</b>**</td></tr>
+  <tr><td colspan="4"><b>MongoDB Storage</b><sup>2</sup></td></tr>
   <tr>
     <td>spring.data.mongodb.host</td>
     <td>URL that points to the MongoDB server</td>
@@ -140,7 +133,7 @@ Here is an explanation provided for all the possibilities on how to tweak and co
   <tr><td colspan="4"><b><a href="./features/compaction">Fragment Compaction</a></b></td></tr>
   <tr>
     <td>ldes-server.compaction-cron</td>
-    <td>Defines how often the Compaction Service will check the fragments ***</td>
+    <td>Defines how often the Compaction Service will check the fragments<sup>3</sup></td>
     <td>No</td>
     <td>0 0 0 * * *</td>
   </tr>
@@ -152,14 +145,14 @@ Here is an explanation provided for all the possibilities on how to tweak and co
   </tr>
   <tr>
     <td>ldes-server.deletion-cron</td>
-    <td>Defines how often the redundant compacted fragments will be checked for deletion ***</td>
+    <td>Defines how often the redundant compacted fragments will be checked for deletion<sup>3</sup></td>
     <td>No</td>
     <td>0 0 0 * * *</td>
   </tr>
   <tr><td colspan="4"><b>Retention (<a href="./configuration/retention-policies">Retention Policies</a>)</b></td></tr>
   <tr>
     <td>ldes-server.retention-cron</td>
-    <td>Defines how often the Retention Service will check the members ***</td>
+    <td>Defines how often the Retention Service will check the members<sup>3</sup></td>
     <td>No</td>
     <td>0 0 0 * * *</td>
   </tr>
@@ -178,27 +171,26 @@ Here is an explanation provided for all the possibilities on how to tweak and co
   </tr>
   <tr>
     <td>ldes-server.admin.port</td>
-    <td>Defines on which port the admin endpoints are available ****</td>
+    <td>Defines on which port the admin endpoints are available<sup>4</sup></td>
     <td>No</td>
     <td>8080</td>
   </tr>
 </tbody>
 </table>
 
-> **Note** *: The specified url will be prefixed by an optional `server.servlet.context-path`
+> **Note** 1: The specified url will be prefixed by an optional `server.servlet.context-path`
 
-> **Note** **: As of this moment the LDES Server only supports a MongoDB implementation. The following properties have
+> **Note** 2: As of this moment the LDES Server only supports a MongoDB implementation. The following properties have
 > to be set to provide connectivity between the server and the database
 
 
-> **Note** ***: Unix usually supports a cron expression of 5 parameters, which excludes seconds. However, the spring
+> **Note** 3: Unix usually supports a cron expression of 5 parameters, which excludes seconds. However, the spring
 > annotation `@Scheduled` adds a 6th parameter to support seconds. The cron schedules are in timezone 'UTC'.
 >
-> More information about this can be found in
->
-the [spring documentation](https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html).
+> More information about this can be found in the [spring documentation]
 
-> **Note** ****: When using the swagger API with separate port bindings, the swagger API will always be available under the admin port.
+> **Note** 4: When using the swagger API with separate port bindings, the swagger API will always be available under
+> the admin port.
 
 ## Docker Compose
 
@@ -228,3 +220,5 @@ networks:
   ldes:
     name: quick_start_network
 ````
+
+[spring documentation]: https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/scheduling/support/CronExpression.html
diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-fragmentisers</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
-        <version>2.14.0</version>
+        <version>2.15.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>ldes-fragmentisers</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
-        <version>2.14.0</version>
+        <version>2.15.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-pagination/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>ldes-fragmentisers</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
-        <version>2.14.0</version>
+        <version>2.15.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-fragmentisers</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
-        <version>2.14.0</version>
+        <version>2.15.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>

diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.vsds</groupId>
         <artifactId>ldes-fragmentisers</artifactId>
-        <version>2.14.0</version>
+        <version>2.15.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ldes-fragmentisers-timebased-hierarchical</artifactId>