Add prod extern sync infrastructure

swisstopo · Nov 21, 2024 · e9c83aa · e9c83aa
2 parents 481fb26 + 2bb41bd
commit e9c83aa
Show file tree

Hide file tree

Showing 42 changed files with 1,161 additions and 66 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -92,3 +92,18 @@ jobs:
       - name: Run view-sync
         working-directory: src/view-sync
         run: docker compose -f docker-compose.yml up --build --exit-code-from view-sync
+
+  extern-sync:
+    runs-on: ubuntu-latest
+    name: Run extern-sync
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup environment
+        working-directory: src/extern-sync
+        run: docker compose -f docker-compose.services.yml up --build --wait
+
+      - name: Run extern-sync
+        working-directory: src/extern-sync
+        run: docker compose -f docker-compose.yml up --build --exit-code-from extern-sync
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -86,6 +86,21 @@ jobs:
             type=edge
             type=semver,pattern=v{{version}},value=${{ env.VERSION }}
 
+      - name: Collect Docker image metadata (extern-sync)
+        id: meta-extern-sync
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.BASE_IMAGE_NAME }}-extern-sync
+          labels: |
+            org.opencontainers.image.created=${{ env.COMMITED_AT }}
+            org.opencontainers.image.version=v${{ env.VERSION }}
+            org.opencontainers.image.maintainer=GeoWerkstatt GmbH <support@geowerkstatt.ch>
+          flavor: |
+            latest=false
+          tags: |
+            type=edge
+            type=semver,pattern=v{{version}},value=${{ env.VERSION }}
+
       - name: Log in to the GitHub container registry
         uses: docker/login-action@v3
         with:
@@ -109,7 +124,7 @@ jobs:
       - name: Build and push Docker image (api)
         uses: docker/build-push-action@v5
         with:
-          context: ./src/api
+          context: ./src
           push: true
           build-args: |
             VERSION=${{ env.VERSION }}
@@ -146,6 +161,19 @@ jobs:
           cache-from: type=registry,ref=${{ env.BASE_IMAGE_NAME }}-view-sync:edge
           cache-to: type=inline
 
+      - name: Build and push Docker image (extern-sync)
+        uses: docker/build-push-action@v5
+        with:
+          context: ./src/extern-sync
+          push: true
+          build-args: |
+            VERSION=${{ env.VERSION }}
+            REVISION=${{ env.REVISION }}
+          tags: ${{ steps.meta-extern-sync.outputs.tags }}
+          labels: ${{ steps.meta-extern-sync.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.BASE_IMAGE_NAME }}-extern-sync:edge
+          cache-to: type=inline
+
       - name: Create GitHub pre-release
         run: |
           gh api \

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -78,6 +78,18 @@ jobs:
           tags: |
             type=semver,pattern=v{{version}}
 
+      - name: Collect Docker image metadata (extern-sync)
+        id: meta-extern-sync
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.BASE_IMAGE_NAME }}-extern-sync
+          labels: |
+            org.opencontainers.image.created=${{ env.COMMITED_AT }}
+            org.opencontainers.image.version=v${{ env.VERSION }}
+            org.opencontainers.image.maintainer=GeoWerkstatt GmbH <support@geowerkstatt.ch>
+          tags: |
+            type=semver,pattern=v{{version}}
+
       - name: Log in to the GitHub container registry
         uses: docker/login-action@v3
         with:
@@ -138,6 +150,19 @@ jobs:
           cache-from: type=registry,ref=${{ env.BASE_IMAGE_NAME }}-view-sync:edge
           cache-to: type=inline
 
+      - name: Build and push Docker image (extern-sync)
+        uses: docker/build-push-action@v5
+        with:
+          context: ./src/extern-sync
+          push: true
+          build-args: |
+            VERSION=${{ env.VERSION }}
+            REVISION=${{ env.REVISION }}
+          tags: ${{ steps.meta-extern-sync.outputs.tags }}
+          labels: ${{ steps.meta-extern-sync.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.BASE_IMAGE_NAME }}-extern-sync:edge
+          cache-to: type=inline
+
   patch-changelog:
     runs-on: ubuntu-latest
     name: Patch CHANGELOG.md and update GitHub release notes

diff --git a/BDMS.sln b/BDMS.sln
@@ -17,6 +17,10 @@ Project("{54A90642-561A-4BB1-A94E-469ADEE60C69}") = "BDMS.Client", "src\client\B
 EndProject
 Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{2B4BC48D-B932-4CB4-B9D6-1336A9F64D79}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BDMS.ExternSync", "src\extern-sync\BDMS.ExternSync.csproj", "{951E6B71-6561-4FA7-9709-5D7F4E1D0DCA}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BDMS.ExternSync.Test", "tests\extern-sync\BDMS.ExternSync.Test.csproj", "{7874F0AF-6311-4A8A-897A-C4DD82A97E6F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -41,6 +45,14 @@ Global
 		{2B4BC48D-B932-4CB4-B9D6-1336A9F64D79}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2B4BC48D-B932-4CB4-B9D6-1336A9F64D79}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2B4BC48D-B932-4CB4-B9D6-1336A9F64D79}.Release|Any CPU.Build.0 = Release|Any CPU
+		{951E6B71-6561-4FA7-9709-5D7F4E1D0DCA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{951E6B71-6561-4FA7-9709-5D7F4E1D0DCA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{951E6B71-6561-4FA7-9709-5D7F4E1D0DCA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{951E6B71-6561-4FA7-9709-5D7F4E1D0DCA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7874F0AF-6311-4A8A-897A-C4DD82A97E6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7874F0AF-6311-4A8A-897A-C4DD82A97E6F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7874F0AF-6311-4A8A-897A-C4DD82A97E6F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7874F0AF-6311-4A8A-897A-C4DD82A97E6F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -93,12 +93,14 @@ services:
       DB_PORT: 5432
   api:
     build:
-      context: ./src/api
+      context: ./src
+      dockerfile: api/Dockerfile
       target: development
     ports:
       - 5000:5000
     volumes:
       - ./src/api:/src
+      - ./src/Directory.Build.props:/Directory.Build.props
       - /src/bin
       - /src/obj
     depends_on:

diff --git a/src/Directory.Build.props b/src/Directory.Build.props
@@ -0,0 +1,28 @@
+<Project>
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <AnalysisLevel>8.0-all</AnalysisLevel>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <Authors>GeoWerkstatt GmbH</Authors>
+    <Company>GeoWerkstatt GmbH</Company>
+    <PackageOutputPath>$(MSBuildThisFileDirectory)/artifacts</PackageOutputPath>
+    <PackageProjectUrl>https://github.com/swisstopo/swissgeol-boreholes-suite</PackageProjectUrl>
+    <RepositoryUrl>https://github.com/swisstopo/swissgeol-boreholes-suite.git</RepositoryUrl>
+    <RepositoryType>git</RepositoryType>
+    <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(MSBuildProjectExtension)' != '.dcproj' And '$(MSBuildProjectExtension)' != '.esproj'">
+    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.507">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>$(AssemblyName).Test</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
+
+</Project>
diff --git a/src/api/BDMS.csproj b/src/api/BDMS.csproj
@@ -1,19 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    <AnalysisLevel>8.0-all</AnalysisLevel>
     <NoWarn>CS1591,CS8618,CS8620</NoWarn>
     <Product>BDMS</Product>
-    <Authors>GeoWerkstatt GmbH</Authors>
-    <Company>GeoWerkstatt GmbH</Company>
-    <PackageOutputPath>$(MSBuildThisFileDirectory)/artifacts</PackageOutputPath>
-    <PackageProjectUrl>https://github.com/swisstopo/swissgeol-boreholes-suite</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/swisstopo/swissgeol-boreholes-suite.git</RepositoryUrl>
-    <RepositoryType>git</RepositoryType>
-    <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     <DockerfileContext>..\..</DockerfileContext>
   </PropertyGroup>
@@ -32,13 +20,6 @@
     <PackageReference Include="Npgsql.NetTopologySuite" Version="8.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
     <PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="6.5.0" />
-    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.507">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
-    </PackageReference>
     <PackageReference Include="Yarp.ReverseProxy" Version="2.1.0" />
-    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-      <_Parameter1>$(AssemblyName).Test</_Parameter1>
-    </AssemblyAttribute>
   </ItemGroup>
 </Project>
diff --git a/src/api/BdmsContext.cs b/src/api/BdmsContext.cs
@@ -2,6 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using Npgsql;
 using System.Security.Claims;
+using static BDMS.BdmsContextConstants;
 
 namespace BDMS;
 
@@ -96,7 +97,7 @@ public async Task<int> UpdateChangeInformationAndSaveChangesAsync(HttpContext ht
     /// <inheritdoc />
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
-        modelBuilder.HasDefaultSchema("bdms");
+        modelBuilder.HasDefaultSchema(BoreholesDatabaseSchemaName);
         modelBuilder.Entity<UserWorkgroupRole>().HasKey(k => new { k.UserId, k.WorkgroupId, k.Role });
         modelBuilder.Entity<TermsAccepted>().HasKey(k => new { k.UserId, k.TermId });
 

diff --git a/src/api/BdmsContextConstants.cs b/src/api/BdmsContextConstants.cs
@@ -0,0 +1,17 @@
+namespace BDMS;
+
+/// <summary>
+/// <see cref="BdmsContext"/> constants.
+/// </summary>
+public static class BdmsContextConstants
+{
+    /// <summary>
+    /// The name of the boreholes database.
+    /// </summary>
+    public const string BoreholesDatabaseName = "bdms";
+
+    /// <summary>
+    /// The name of the boreholes database schema.
+    /// </summary>
+    public const string BoreholesDatabaseSchemaName = "bdms";
+}
diff --git a/src/api/BdmsContextExtensions.cs b/src/api/BdmsContextExtensions.cs
@@ -4,17 +4,29 @@
 using Microsoft.Data.SqlClient;
 using Microsoft.EntityFrameworkCore;
 using NetTopologySuite.Geometries;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure;
 using System.Collections.ObjectModel;
 using System.Globalization;
+using static BDMS.BdmsContextConstants;
 
 namespace BDMS;
 
 #pragma warning disable CA1505
 /// <summary>
-/// Contains extensions methods for the BDMS db context.
+/// Contains extensions methods for the <see cref="BdmsContext"/>.
 /// </summary>
 public static class BdmsContextExtensions
 {
+    /// <summary>
+    /// Sets the default <see cref="NpgsqlDbContextOptionsBuilder"/> options for the boreholes database context.
+    /// </summary>
+    public static void SetDbContextOptions(this NpgsqlDbContextOptionsBuilder options)
+    {
+        options.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery);
+        options.UseNetTopologySuite();
+        options.MigrationsHistoryTable("__EFMigrationsHistory", BoreholesDatabaseName);
+    }
+
     /// <summary>
     /// Seed test data but only if the database is not yet seeded.
     /// </summary>

diff --git a/src/api/Dockerfile b/src/api/Dockerfile
@@ -3,13 +3,16 @@ ENV ASPNETCORE_ENVIRONMENT=Development
 WORKDIR /src
 
 # Install missing packages
-RUN apt-get -y update
-RUN apt-get -y install git vim curl htop
-RUN dotnet tool install --global dotnet-ef --version 8.0.0
+RUN \
+  apt-get -y update && \
+  apt-get -y install curl git htop vim && \
+  rm -rf /var/lib/apt/lists/* && \
+  dotnet tool install --global dotnet-ef --version 8.0.0
+
 ENV PATH=$PATH:/root/.dotnet/tools
 
 # Restore dependencies and tools
-COPY BDMS.csproj .
+COPY api/BDMS.csproj Directory.Build.props ./
 RUN dotnet restore
 
 ENTRYPOINT ["dotnet", "watch", "run", "--no-launch-profile"]
@@ -20,11 +23,11 @@ ARG REVISION
 WORKDIR /src
 
 # Restore dependencies and tools
-COPY BDMS.csproj .
+COPY api/BDMS.csproj Directory.Build.props ./
 RUN dotnet restore
 
 # Create optimized production build
-COPY . .
+COPY api/ Directory.Build.props ./
 RUN dotnet publish \
   -c Release \
   -p:UseAppHost=false \

diff --git a/src/api/Program.cs b/src/api/Program.cs
@@ -52,12 +52,7 @@
 builder.Services.AddHttpClient();
 
 var connectionString = builder.Configuration.GetConnectionString(nameof(BdmsContext));
-builder.Services.AddNpgsql<BdmsContext>(connectionString, options =>
-{
-    options.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery);
-    options.UseNetTopologySuite();
-    options.MigrationsHistoryTable("__EFMigrationsHistory", "bdms");
-});
+builder.Services.AddNpgsql<BdmsContext>(connectionString, BdmsContextExtensions.SetDbContextOptions);
 
 builder.Services.Configure<RouteOptions>(options => options.LowercaseUrls = true);
 builder.Services.AddSwaggerGen(options =>

diff --git a/src/extern-sync/BDMS.ExternSync.csproj b/src/extern-sync/BDMS.ExternSync.csproj
@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <Product>BDMS.ExternSync</Product>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="docker-compose.services.yml" />
+    <None Remove="docker-compose.yml" />
+    <None Remove="Dockerfile" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Npgsql.DependencyInjection" Version="8.0.0" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\api\BDMS.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+</Project>
diff --git a/src/extern-sync/Dockerfile b/src/extern-sync/Dockerfile
@@ -0,0 +1,33 @@
+# syntax=docker/dockerfile:1.7-labs
+
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+ARG VERSION
+ARG REVISION
+WORKDIR /src
+
+# Restore dependencies and tools
+COPY --parents "extern-sync/BDMS.ExternSync.csproj" "api/BDMS.csproj" "Directory.Build.props" ./
+RUN dotnet restore "./extern-sync/BDMS.ExternSync.csproj"
+
+# Create optimized production build
+COPY --parents "extern-sync/" "api/" ./
+RUN dotnet publish "./extern-sync/BDMS.ExternSync.csproj" \
+  -c Release \
+  -p:UseAppHost=false \
+  -p:VersionPrefix=${VERSION} \
+  -p:SourceRevisionId=${REVISION} \
+  -o /app/publish
+
+FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS deploy
+ENV ASPNETCORE_ENVIRONMENT=Production
+WORKDIR /app
+
+# Set default locale
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+COPY --from=build /app/publish .
+
+# Switch to the non-root user 'app' defined in the base image
+USER $APP_UID
+ENTRYPOINT ["dotnet", "BDMS.ExternSync.dll"]