A start

sillsdev · Nov 28, 2024 · e300f33 · e300f33
1 parent 03e4716
commit e300f33
Show file tree

Hide file tree

Showing 27 changed files with 198 additions and 82 deletions.
diff --git a/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs b/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs
@@ -657,7 +657,6 @@ CancellationToken cancellationToken
                     new CorpusFile
                     {
                         Id = fileConfig.FileId,
-                        Filename = result.Message.Filename,
                         TextId = fileConfig.TextId ?? result.Message.Name,
                         Format = result.Message.Format
                     }

diff --git a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs
diff --git a/src/Serval/src/Serval.Assessment/Services/EngineService.cs b/src/Serval/src/Serval.Assessment/Services/EngineService.cs
@@ -227,13 +227,13 @@ private V1.Corpus Map(Models.Corpus source)
         return new V1.Corpus { Language = source.Language, Files = { source.Files.Select(Map) } };
     }
 
-    private V1.CorpusFile Map(Models.CorpusFile source)
+    private V1.CorpusFile Map(Shared.Models.CorpusFile source)
     {
         return new V1.CorpusFile
         {
             TextId = source.TextId,
             Format = (V1.FileFormat)source.Format,
-            Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename)
+            Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.GetFilename())
         };
     }
 }
diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs
@@ -9293,15 +9293,15 @@ public partial class CorpusFile
     {
         [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)]
         [System.ComponentModel.DataAnnotations.Required]
-        public DataFile File { get; set; } = new DataFile();
+        public DataFileReference File { get; set; } = new DataFileReference();
 
         [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string? TextId { get; set; } = default!;
 
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
-    public partial class DataFile
+    public partial class DataFileReference
     {
         [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)]
         [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
@@ -9319,9 +9319,6 @@ public partial class DataFile
         [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]
         public FileFormat Format { get; set; } = default!;
 
-        [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)]
-        public int Revision { get; set; } = default!;
-
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
@@ -9364,6 +9361,30 @@ public partial class CorpusFileConfig
 
     }
 
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class DataFile
+    {
+        [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public string Id { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public string Url { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string? Name { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("format", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]
+        public FileFormat Format { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)]
+        public int Revision { get; set; } = default!;
+
+    }
+
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class TranslationEngine
     {

diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs
@@ -20,16 +20,11 @@ await context.RespondAsync(
                     Name = corpus.Name,
                     Language = corpus.Language,
                     Files = corpus
-                        .Files.Select(f => new CorpusFileResult
+                        .Files.Select(f => new Shared.Models.CorpusFile
                         {
+                            Id = f.FileReference.Id,
                             TextId = f.TextId!,
-                            File = new DataFileResult
-                            {
-                                DataFileId = f.File.Id,
-                                Filename = f.File.Filename,
-                                Format = f.File.Format,
-                                Name = f.File.Name
-                            }
+                            Format = f.FileReference.Format
                         })
                         .ToList()
                 }

diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs
@@ -2,6 +2,6 @@ namespace Serval.DataFiles.Contracts;
 
 public record CorpusFileDto
 {
-    public required DataFileDto File { get; init; }
+    public required DataFileReferenceDto File { get; init; }
     public string? TextId { get; init; }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs
@@ -0,0 +1,9 @@
+namespace Serval.DataFiles.Contracts;
+
+public record DataFileReferenceDto
+{
+    public required string Id { get; init; }
+    public required string Url { get; init; }
+    public string? Name { get; init; }
+    public required FileFormat Format { get; init; }
+}
diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs
@@ -166,18 +166,18 @@ private async Task<Corpus> MapAsync(CorpusConfigDto corpusConfig, string id, Can
         };
     }
 
-    private async Task<IReadOnlyList<CorpusFile>> MapAsync(
+    private async Task<IReadOnlyList<Models.CorpusFile>> MapAsync(
         IReadOnlyList<CorpusFileConfigDto> files,
         CancellationToken cancellationToken
     )
     {
-        var dataFiles = new List<CorpusFile>();
+        var dataFiles = new List<Models.CorpusFile>();
         foreach (CorpusFileConfigDto file in files)
         {
             DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken);
             if (dataFile == null)
                 throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist.");
-            dataFiles.Add(new CorpusFile { File = dataFile, TextId = file.TextId });
+            dataFiles.Add(new Models.CorpusFile { FileReference = Map(dataFile), TextId = file.TextId });
         }
         return dataFiles;
     }
@@ -195,20 +195,29 @@ private CorpusDto Map(Corpus source)
         };
     }
 
-    private CorpusFileDto Map(CorpusFile source)
+    private CorpusFileDto Map(Models.CorpusFile source)
     {
-        return new CorpusFileDto { File = Map(source.File), TextId = source.TextId };
+        return new CorpusFileDto { File = Map(source.FileReference), TextId = source.TextId };
     }
 
-    private DataFileDto Map(DataFile source)
+    private DataFileReferenceDto Map(DataFileReference source)
     {
-        return new DataFileDto
+        return new DataFileReferenceDto
         {
             Id = source.Id,
             Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }),
             Name = source.Name,
             Format = source.Format,
-            Revision = source.Revision
+        };
+    }
+
+    private static DataFileReference Map(DataFile source)
+    {
+        return new DataFileReference
+        {
+            Id = source.Id,
+            Format = source.Format,
+            Name = source.Name
         };
     }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs
@@ -2,6 +2,6 @@ namespace Serval.DataFiles.Models;
 
 public record CorpusFile
 {
-    public required DataFile File { get; init; }
+    public required DataFileReference FileReference { get; init; }
     public string? TextId { get; init; }
 }
diff --git a/src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs b/src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs
@@ -0,0 +1,8 @@
+namespace Serval.DataFiles.Models;
+
+public record DataFileReference
+{
+    public string Id { get; set; } = "";
+    public required string Name { get; init; }
+    public required FileFormat Format { get; init; }
+}
diff --git a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs
@@ -12,7 +12,7 @@ public async Task<Corpus> GetAsync(string id, string owner, CancellationToken ca
 
     public async Task<Corpus> UpdateAsync(
         string id,
-        IReadOnlyList<CorpusFile> files,
+        IReadOnlyList<Models.CorpusFile> files,
         CancellationToken cancellationToken = default
     )
     {

diff --git a/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs
@@ -6,6 +6,10 @@ public interface ICorpusService
     Task<Corpus> GetAsync(string id, CancellationToken cancellationToken = default);
     Task<Corpus> GetAsync(string id, string owner, CancellationToken cancellationToken = default);
     Task<Corpus> CreateAsync(Corpus corpus, CancellationToken cancellationToken = default);
-    Task<Corpus> UpdateAsync(string id, IReadOnlyList<CorpusFile> files, CancellationToken cancellationToken = default);
+    Task<Corpus> UpdateAsync(
+        string id,
+        IReadOnlyList<Models.CorpusFile> files,
+        CancellationToken cancellationToken = default
+    );
     Task DeleteAsync(string id, CancellationToken cancellationToken = default);
 }
diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs
@@ -5,5 +5,5 @@ public record CorpusResult
     public required string CorpusId { get; init; }
     public required string Language { get; init; }
     public string? Name { get; init; }
-    public required IReadOnlyList<CorpusFileResult> Files { get; set; }
+    public required IReadOnlyList<CorpusFile> Files { get; set; }
 }
diff --git a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs
@@ -0,0 +1,35 @@
+namespace Serval.Shared.Models;
+
+public record CorpusFile
+{
+    public required string Id { get; set; }
+    public required FileFormat Format { get; set; }
+    public required string TextId { get; set; }
+
+    private string? _filename;
+
+    public async Task<string> PopulateFilenameAsync(
+        IRequestClient<GetDataFile> getDataFileClient,
+        string owner,
+        CancellationToken cancellationToken
+    )
+    {
+        Response<DataFileResult, DataFileNotFound> response = await getDataFileClient.GetResponse<
+            DataFileResult,
+            DataFileNotFound
+        >(new GetDataFile { DataFileId = Id, Owner = owner }, cancellationToken);
+        if (response.Is(out Response<DataFileResult>? result))
+        {
+            _filename = result.Message.Filename;
+        }
+        throw new InvalidOperationException($"The data file {Id} cannot be found.");
+    }
+
+    public string GetFilename()
+    {
+        return _filename
+            ?? throw new InvalidOperationException(
+                "The filename has not been populated.  It is not stored in the database."
+            );
+    }
+}
diff --git a/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs
@@ -0,0 +1,20 @@
+namespace Serval.Shared.Models;
+
+public record MonolingualCorpus
+{
+    public required string Id { get; set; }
+    public string? Name { get; set; }
+    public required string Language { get; set; }
+    public required IReadOnlyList<CorpusFile> Files { get; set; }
+
+    public async Task PopulateFilenamesAsync(
+        IRequestClient<GetDataFile> getDataFileClient,
+        string owner,
+        CancellationToken cancellationToken
+    )
+    {
+        await Task.WhenAll(
+            Files.Select(file => file.PopulateFilenameAsync(getDataFileClient, owner, cancellationToken))
+        );
+    }
+}
diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj
@@ -21,6 +21,7 @@
 		<PackageReference Include="Grpc.Net.ClientFactory" Version="2.65.0" />
 		<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
+		<PackageReference Include="MassTransit" Version="8.0.14" />
 	</ItemGroup>
 
 	<ItemGroup>

diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs
@@ -3,6 +3,7 @@
 global using System.Text.Json.Serialization;
 global using Grpc.Core;
 global using Grpc.Net.ClientFactory;
+global using MassTransit;
 global using Microsoft.AspNetCore.Authorization;
 global using Microsoft.AspNetCore.Http;
 global using Microsoft.AspNetCore.Mvc;
@@ -12,6 +13,7 @@
 global using Microsoft.Extensions.Logging;
 global using Microsoft.Extensions.Options;
 global using Serval.Shared.Configuration;
+global using Serval.Shared.Contracts;
 global using Serval.Shared.Models;
 global using Serval.Shared.Services;
 global using Serval.Shared.Utils;

diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs
@@ -852,6 +852,7 @@ CancellationToken cancellationToken
     /// <param name="corpusId">The corpus id or parallel corpus id</param>
     /// <param name="textId">The text id</param>
     /// <param name="textOrigin">The source[s] of the data to populate the USFM file with.</param>
+    /// <param name="getDataFileClient">The data file client</param>
     /// <param name="cancellationToken"></param>
     /// <response code="200">The book in USFM format</response>
     /// <response code="204">The specified book does not exist in the source or target corpus.</response>
@@ -878,11 +879,13 @@ public async Task<IActionResult> GetPretranslatedUsfmAsync(
         [NotNull] string textId,
         [FromQuery(Name = "text-origin")] PretranslationUsfmTextOrigin? textOrigin,
         [FromQuery] PretranslationUsfmTemplate? template,
+        [FromServices] IRequestClient<GetDataFile> getDataFileClient,
         CancellationToken cancellationToken
     )
     {
         Engine engine = await _engineService.GetAsync(id, cancellationToken);
         await AuthorizeAsync(engine);
+        await engine.PopulateFilenamesAsync(getDataFileClient, cancellationToken);
         if (!engine.Corpora.Any(c => c.Id == corpusId) && !engine.ParallelCorpora.Any(c => c.Id == corpusId))
             return NotFound();
         if (engine.ModelRevision == 0)
@@ -1031,6 +1034,7 @@ CancellationToken cancellationToken
     /// </remarks>
     /// <param name="id">The translation engine id</param>
     /// <param name="buildConfig">The build config (see remarks)</param>
+    /// <param name="getDataFileClient">The data file client</param>
     /// <param name="cancellationToken"></param>
     /// <response code="201">The new build job</response>
     /// <response code="400">The build configuration was invalid.</response>
@@ -1051,13 +1055,15 @@ CancellationToken cancellationToken
     public async Task<ActionResult<TranslationBuildDto>> StartBuildAsync(
         [NotNull] string id,
         [FromBody] TranslationBuildConfigDto buildConfig,
+        [FromServices] IRequestClient<GetDataFile> getDataFileClient,
         CancellationToken cancellationToken
     )
     {
         string deploymentVersion = _configuration.GetValue<string>("deploymentVersion") ?? "Unknown";
 
         Engine engine = await _engineService.GetAsync(id, cancellationToken);
         await AuthorizeAsync(engine);
+        await engine.PopulateFilenamesAsync(getDataFileClient, cancellationToken);
         Build build = Map(engine, buildConfig, deploymentVersion);
 
         await _engineService.StartBuildAsync(build, cancellationToken);
@@ -1220,7 +1226,7 @@ CancellationToken cancellationToken
     }
 
     private async Task<ParallelCorpus> MapAsync(
-        IRequestClient<GetCorpus> getDataFileClient,
+        IRequestClient<GetCorpus> getCorpusClient,
         string corpusId,
         TranslationParallelCorpusConfigDto source,
         CancellationToken cancellationToken
@@ -1229,8 +1235,8 @@ CancellationToken cancellationToken
         return new ParallelCorpus
         {
             Id = corpusId,
-            SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken),
-            TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken)
+            SourceCorpora = await MapAsync(getCorpusClient, source.SourceCorpusIds, cancellationToken),
+            TargetCorpora = await MapAsync(getCorpusClient, source.TargetCorpusIds, cancellationToken)
         };
     }
 
@@ -1253,7 +1259,6 @@ CancellationToken cancellationToken
                     new CorpusFile
                     {
                         Id = fileConfig.FileId,
-                        Filename = result.Message.Filename,
                         TextId = fileConfig.TextId ?? result.Message.Name,
                         Format = result.Message.Format
                     }
@@ -1291,9 +1296,8 @@ CancellationToken cancellationToken
                         Files = result
                             .Message.Files.Select(f => new CorpusFile
                             {
-                                Id = f.File.DataFileId,
-                                Filename = f.File.Filename,
-                                Format = f.File.Format,
+                                Id = f.Id,
+                                Format = f.Format,
                                 TextId = f.TextId
                             })
                             .ToList(),