From ede3bb689dc45c7faa41214857f80d0f6ea31c12 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 28 Nov 2024 07:21:52 -0500 Subject: [PATCH] Remove filename from corpus data structure Repopulate filename when building and creating USFM files --- .../AssessmentEnginesController.cs | 1 - .../Serval.Assessment/Models/CorpusFile.cs | 9 -- .../Services/EngineService.cs | 4 +- src/Serval/src/Serval.Client/Client.g.cs | 31 +++- .../Consumers/GetCorpusConsumer.cs | 11 +- .../Contracts/CorpusFileDto.cs | 2 +- .../Contracts/DataFileReferenceDto.cs | 9 ++ .../Controllers/CorporaController.cs | 25 ++- .../src/Serval.DataFiles/Models/CorpusFile.cs | 2 +- .../Models/DataFileReference.cs | 8 + .../Services/CorpusService.cs | 2 +- .../Services/ICorpusService.cs | 6 +- .../Serval.Shared/Contracts/CorpusResult.cs | 2 +- .../src/Serval.Shared/Models/CorpusFile.cs | 43 ++++++ .../Serval.Shared/Models/MonolingualCorpus.cs | 20 +++ .../src/Serval.Shared/Serval.Shared.csproj | 1 + src/Serval/src/Serval.Shared/Usings.cs | 2 + .../TranslationEnginesController.cs | 12 +- .../src/Serval.Translation/Models/Corpus.cs | 14 ++ .../Serval.Translation/Models/CorpusFile.cs | 9 -- .../src/Serval.Translation/Models/Engine.cs | 13 ++ .../Models/MonolingualCorpus.cs | 9 -- .../Models/ParallelCorpus.cs | 14 ++ .../Services/EngineService.cs | 21 +-- .../Services/PretranslationService.cs | 13 +- .../AssessmentEngineTests.cs | 1 - .../TranslationEngineTests.cs | 34 ++-- .../Services/CorpusServiceTests.cs | 6 +- .../Services/EngineServiceTests.cs | 145 ++++++++---------- .../Services/PretranslationServiceTests.cs | 17 +- .../test/Serval.Translation.Tests/Usings.cs | 1 + 31 files changed, 292 insertions(+), 195 deletions(-) delete mode 100644 src/Serval/src/Serval.Assessment/Models/CorpusFile.cs create mode 100644 src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs create mode 100644 src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs create mode 100644 src/Serval/src/Serval.Shared/Models/CorpusFile.cs create mode 100644 src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs delete mode 100644 src/Serval/src/Serval.Translation/Models/CorpusFile.cs delete mode 100644 src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs diff --git a/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs b/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs index 459d3b34..ff9b99ed 100644 --- a/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs +++ b/src/Serval/src/Serval.Assessment/Controllers/AssessmentEnginesController.cs @@ -657,7 +657,6 @@ CancellationToken cancellationToken new CorpusFile { Id = fileConfig.FileId, - Filename = result.Message.Filename, TextId = fileConfig.TextId ?? result.Message.Name, Format = result.Message.Format } diff --git a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs b/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs deleted file mode 100644 index fa491558..00000000 --- a/src/Serval/src/Serval.Assessment/Models/CorpusFile.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Assessment.Models; - -public record CorpusFile -{ - public required string Id { get; init; } - public required string Filename { get; init; } - public required FileFormat Format { get; init; } - public required string TextId { get; init; } -} diff --git a/src/Serval/src/Serval.Assessment/Services/EngineService.cs b/src/Serval/src/Serval.Assessment/Services/EngineService.cs index 01ced93a..972edccf 100644 --- a/src/Serval/src/Serval.Assessment/Services/EngineService.cs +++ b/src/Serval/src/Serval.Assessment/Services/EngineService.cs @@ -227,13 +227,13 @@ private V1.Corpus Map(Models.Corpus source) return new V1.Corpus { Language = source.Language, Files = { source.Files.Select(Map) } }; } - private V1.CorpusFile Map(Models.CorpusFile source) + private V1.CorpusFile Map(Shared.Models.CorpusFile source) { return new V1.CorpusFile { TextId = source.TextId, Format = (V1.FileFormat)source.Format, - Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename) + Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.GetFilename()) }; } } diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index ee4ce398..8b8e2615 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -9293,7 +9293,7 @@ public partial class CorpusFile { [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public DataFile File { get; set; } = new DataFile(); + public DataFileReference File { get; set; } = new DataFileReference(); [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? TextId { get; set; } = default!; @@ -9301,7 +9301,7 @@ public partial class CorpusFile } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class DataFile + public partial class DataFileReference { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -9319,9 +9319,6 @@ public partial class DataFile [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] public FileFormat Format { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] - public int Revision { get; set; } = default!; - } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] @@ -9364,6 +9361,30 @@ public partial class CorpusFileConfig } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class DataFile + { + [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Id { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("url", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Url { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Name { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("format", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public FileFormat Format { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("revision", Required = Newtonsoft.Json.Required.Always)] + public int Revision { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationEngine { diff --git a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs index c369d528..dd6b806b 100644 --- a/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs +++ b/src/Serval/src/Serval.DataFiles/Consumers/GetCorpusConsumer.cs @@ -20,16 +20,11 @@ await context.RespondAsync( Name = corpus.Name, Language = corpus.Language, Files = corpus - .Files.Select(f => new CorpusFileResult + .Files.Select(f => new Shared.Models.CorpusFile { + Id = f.FileReference.Id, TextId = f.TextId!, - File = new DataFileResult - { - DataFileId = f.File.Id, - Filename = f.File.Filename, - Format = f.File.Format, - Name = f.File.Name - } + Format = f.FileReference.Format }) .ToList() } diff --git a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs index d2d175be..596c9599 100644 --- a/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs +++ b/src/Serval/src/Serval.DataFiles/Contracts/CorpusFileDto.cs @@ -2,6 +2,6 @@ namespace Serval.DataFiles.Contracts; public record CorpusFileDto { - public required DataFileDto File { get; init; } + public required DataFileReferenceDto File { get; init; } public string? TextId { get; init; } } diff --git a/src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs b/src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs new file mode 100644 index 00000000..e996ddf3 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Contracts/DataFileReferenceDto.cs @@ -0,0 +1,9 @@ +namespace Serval.DataFiles.Contracts; + +public record DataFileReferenceDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public string? Name { get; init; } + public required FileFormat Format { get; init; } +} diff --git a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs index 29bf041e..1b4b307a 100644 --- a/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs +++ b/src/Serval/src/Serval.DataFiles/Controllers/CorporaController.cs @@ -166,18 +166,18 @@ private async Task MapAsync(CorpusConfigDto corpusConfig, string id, Can }; } - private async Task> MapAsync( + private async Task> MapAsync( IReadOnlyList files, CancellationToken cancellationToken ) { - var dataFiles = new List(); + var dataFiles = new List(); foreach (CorpusFileConfigDto file in files) { DataFile? dataFile = await _dataFileService.GetAsync(file.FileId, cancellationToken); if (dataFile == null) throw new InvalidOperationException($"DataFile with id {file.FileId} does not exist."); - dataFiles.Add(new CorpusFile { File = dataFile, TextId = file.TextId }); + dataFiles.Add(new Models.CorpusFile { FileReference = Map(dataFile), TextId = file.TextId }); } return dataFiles; } @@ -195,20 +195,29 @@ private CorpusDto Map(Corpus source) }; } - private CorpusFileDto Map(CorpusFile source) + private CorpusFileDto Map(Models.CorpusFile source) { - return new CorpusFileDto { File = Map(source.File), TextId = source.TextId }; + return new CorpusFileDto { File = Map(source.FileReference), TextId = source.TextId }; } - private DataFileDto Map(DataFile source) + private DataFileReferenceDto Map(DataFileReference source) { - return new DataFileDto + return new DataFileReferenceDto { Id = source.Id, Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }), Name = source.Name, Format = source.Format, - Revision = source.Revision + }; + } + + private static DataFileReference Map(DataFile source) + { + return new DataFileReference + { + Id = source.Id, + Format = source.Format, + Name = source.Name }; } } diff --git a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs index a4311e39..a08a5578 100644 --- a/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.DataFiles/Models/CorpusFile.cs @@ -2,6 +2,6 @@ namespace Serval.DataFiles.Models; public record CorpusFile { - public required DataFile File { get; init; } + public required DataFileReference FileReference { get; init; } public string? TextId { get; init; } } diff --git a/src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs b/src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs new file mode 100644 index 00000000..ec59f865 --- /dev/null +++ b/src/Serval/src/Serval.DataFiles/Models/DataFileReference.cs @@ -0,0 +1,8 @@ +namespace Serval.DataFiles.Models; + +public record DataFileReference +{ + public string Id { get; set; } = ""; + public required string Name { get; init; } + public required FileFormat Format { get; init; } +} diff --git a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs index f5b8e4b6..a092f45b 100644 --- a/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/CorpusService.cs @@ -12,7 +12,7 @@ public async Task GetAsync(string id, string owner, CancellationToken ca public async Task UpdateAsync( string id, - IReadOnlyList files, + IReadOnlyList files, CancellationToken cancellationToken = default ) { diff --git a/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs b/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs index a4f0e242..d5aa3645 100644 --- a/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs +++ b/src/Serval/src/Serval.DataFiles/Services/ICorpusService.cs @@ -6,6 +6,10 @@ public interface ICorpusService Task GetAsync(string id, CancellationToken cancellationToken = default); Task GetAsync(string id, string owner, CancellationToken cancellationToken = default); Task CreateAsync(Corpus corpus, CancellationToken cancellationToken = default); - Task UpdateAsync(string id, IReadOnlyList files, CancellationToken cancellationToken = default); + Task UpdateAsync( + string id, + IReadOnlyList files, + CancellationToken cancellationToken = default + ); Task DeleteAsync(string id, CancellationToken cancellationToken = default); } diff --git a/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs b/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs index 0c0f8380..1e2d5376 100644 --- a/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs +++ b/src/Serval/src/Serval.Shared/Contracts/CorpusResult.cs @@ -5,5 +5,5 @@ public record CorpusResult public required string CorpusId { get; init; } public required string Language { get; init; } public string? Name { get; init; } - public required IReadOnlyList Files { get; set; } + public required IReadOnlyList Files { get; set; } } diff --git a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs new file mode 100644 index 00000000..3b83c257 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs @@ -0,0 +1,43 @@ +namespace Serval.Shared.Models; + +public record CorpusFile +{ + public required string Id { get; set; } + public required FileFormat Format { get; set; } + public required string TextId { get; set; } + + private string? _filename; + + public async Task PopulateFilenameAsync( + IRequestClient getDataFileClient, + string owner, + CancellationToken cancellationToken + ) + { + Response response = await getDataFileClient.GetResponse< + DataFileResult, + DataFileNotFound + >(new GetDataFile { DataFileId = Id, Owner = owner }, cancellationToken); + if (response.Is(out Response? result)) + { + _filename = result.Message.Filename; + } + else + { + throw new InvalidOperationException($"The data file {Id} cannot be found."); + } + } + + public void SetFilename(string filename) + { + _filename = filename; + } + + public string GetFilename() + { + return _filename + ?? throw new InvalidOperationException( + "The filename has not been populated. It is not stored in the database." + ); + } +} diff --git a/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs new file mode 100644 index 00000000..d2c9ab4f --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs @@ -0,0 +1,20 @@ +namespace Serval.Shared.Models; + +public record MonolingualCorpus +{ + public required string Id { get; set; } + public string? Name { get; set; } + public required string Language { get; set; } + public required IReadOnlyList Files { get; set; } + + public async Task PopulateFilenamesAsync( + IRequestClient getDataFileClient, + string owner, + CancellationToken cancellationToken + ) + { + await Task.WhenAll( + Files.Select(file => file.PopulateFilenameAsync(getDataFileClient, owner, cancellationToken)) + ); + } +} diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index f2607b7b..b9d5a90c 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -21,6 +21,7 @@ + diff --git a/src/Serval/src/Serval.Shared/Usings.cs b/src/Serval/src/Serval.Shared/Usings.cs index 3e84144f..901c11a6 100644 --- a/src/Serval/src/Serval.Shared/Usings.cs +++ b/src/Serval/src/Serval.Shared/Usings.cs @@ -3,6 +3,7 @@ global using System.Text.Json.Serialization; global using Grpc.Core; global using Grpc.Net.ClientFactory; +global using MassTransit; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; @@ -12,6 +13,7 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Models; global using Serval.Shared.Services; global using Serval.Shared.Utils; diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 9b735a01..ab56c599 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1220,7 +1220,7 @@ CancellationToken cancellationToken } private async Task MapAsync( - IRequestClient getDataFileClient, + IRequestClient getCorpusClient, string corpusId, TranslationParallelCorpusConfigDto source, CancellationToken cancellationToken @@ -1229,8 +1229,8 @@ CancellationToken cancellationToken return new ParallelCorpus { Id = corpusId, - SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken), - TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken) + SourceCorpora = await MapAsync(getCorpusClient, source.SourceCorpusIds, cancellationToken), + TargetCorpora = await MapAsync(getCorpusClient, source.TargetCorpusIds, cancellationToken) }; } @@ -1253,7 +1253,6 @@ CancellationToken cancellationToken new CorpusFile { Id = fileConfig.FileId, - Filename = result.Message.Filename, TextId = fileConfig.TextId ?? result.Message.Name, Format = result.Message.Format } @@ -1291,9 +1290,8 @@ CancellationToken cancellationToken Files = result .Message.Files.Select(f => new CorpusFile { - Id = f.File.DataFileId, - Filename = f.File.Filename, - Format = f.File.Format, + Id = f.Id, + Format = f.Format, TextId = f.TextId }) .ToList(), diff --git a/src/Serval/src/Serval.Translation/Models/Corpus.cs b/src/Serval/src/Serval.Translation/Models/Corpus.cs index e8e3a4d4..7019dcbc 100644 --- a/src/Serval/src/Serval.Translation/Models/Corpus.cs +++ b/src/Serval/src/Serval.Translation/Models/Corpus.cs @@ -8,4 +8,18 @@ public record Corpus public required string TargetLanguage { get; set; } public required IReadOnlyList SourceFiles { get; set; } public required IReadOnlyList TargetFiles { get; set; } + + public async Task PopulateFilenamesAsync( + IRequestClient getDataFileClient, + string owner, + CancellationToken cancellationToken + ) + { + await Task.WhenAll( + SourceFiles.Select(file => file.PopulateFilenameAsync(getDataFileClient, owner, cancellationToken)) + ); + await Task.WhenAll( + TargetFiles.Select(file => file.PopulateFilenameAsync(getDataFileClient, owner, cancellationToken)) + ); + } } diff --git a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs b/src/Serval/src/Serval.Translation/Models/CorpusFile.cs deleted file mode 100644 index 2672ba56..00000000 --- a/src/Serval/src/Serval.Translation/Models/CorpusFile.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Translation.Models; - -public record CorpusFile -{ - public required string Id { get; set; } - public required string Filename { get; set; } - public required FileFormat Format { get; set; } - public required string TextId { get; set; } -} diff --git a/src/Serval/src/Serval.Translation/Models/Engine.cs b/src/Serval/src/Serval.Translation/Models/Engine.cs index b4d0f55b..f3f84946 100644 --- a/src/Serval/src/Serval.Translation/Models/Engine.cs +++ b/src/Serval/src/Serval.Translation/Models/Engine.cs @@ -16,4 +16,17 @@ public record Engine : IOwnedEntity public int ModelRevision { get; init; } public double Confidence { get; init; } public int CorpusSize { get; init; } + + public async Task PopulateFilenamesAsync( + IRequestClient getDataFileClient, + CancellationToken cancellationToken + ) + { + await Task.WhenAll( + Corpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, Owner, cancellationToken)) + ); + await Task.WhenAll( + ParallelCorpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, Owner, cancellationToken)) + ); + } } diff --git a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs deleted file mode 100644 index 0762e878..00000000 --- a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Translation.Models; - -public record MonolingualCorpus -{ - public required string Id { get; set; } - public string? Name { get; set; } - public required string Language { get; set; } - public required IReadOnlyList Files { get; set; } -} diff --git a/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs index 0fd059c7..a1a02ffd 100644 --- a/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs +++ b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs @@ -5,4 +5,18 @@ public record ParallelCorpus public required string Id { get; set; } public IReadOnlyList SourceCorpora { get; set; } = new List(); public IReadOnlyList TargetCorpora { get; set; } = new List(); + + public async Task PopulateFilenamesAsync( + IRequestClient getDataFileClient, + string owner, + CancellationToken cancellationToken + ) + { + await Task.WhenAll( + SourceCorpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, owner, cancellationToken)) + ); + await Task.WhenAll( + TargetCorpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, owner, cancellationToken)) + ); + } } diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index a8bb3a05..12978f3f 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -12,7 +12,8 @@ public class EngineService( IOptionsMonitor dataFileOptions, IDataAccessContext dataAccessContext, ILoggerFactory loggerFactory, - IScriptureDataFileService scriptureDataFileService + IScriptureDataFileService scriptureDataFileService, + IRequestClient getDataFileClient ) : OwnedEntityServiceBase(engines), IEngineService { private readonly IRepository _builds = builds; @@ -23,6 +24,7 @@ IScriptureDataFileService scriptureDataFileService private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = loggerFactory.CreateLogger(); private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + private readonly IRequestClient _getDataFileClient = getDataFileClient; public async Task TranslateAsync( string engineId, @@ -217,6 +219,7 @@ private Dictionary> GetChapters(string fileLocation, string sc public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) { Engine engine = await GetAsync(build.EngineRef, cancellationToken); + await engine.PopulateFilenamesAsync(_getDataFileClient, cancellationToken); await _builds.InsertAsync(build, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = @@ -385,11 +388,11 @@ public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationTo return Entities.UpdateAsync(engineId, u => u.Add(e => e.Corpora, corpus), cancellationToken: cancellationToken); } - public async Task UpdateCorpusAsync( + public async Task UpdateCorpusAsync( string engineId, string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + IReadOnlyList? sourceFiles, + IReadOnlyList? targetFiles, CancellationToken cancellationToken = default ) { @@ -465,8 +468,8 @@ public Task AddParallelCorpusAsync( public async Task UpdateParallelCorpusAsync( string engineId, string parallelCorpusId, - IReadOnlyList? sourceCorpora, - IReadOnlyList? targetCorpora, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { @@ -790,7 +793,7 @@ bool pretranslateOnAllCorpora } private V1.MonolingualCorpus Map( - Models.MonolingualCorpus inputCorpus, + Shared.Models.MonolingualCorpus inputCorpus, ParallelCorpusFilter? trainingFilter, ParallelCorpusFilter? pretranslateFilter, string? referenceFileLocation, @@ -880,13 +883,13 @@ pretranslateFilter is not null return returnCorpus; } - private V1.CorpusFile Map(Models.CorpusFile source) + private V1.CorpusFile Map(Shared.Models.CorpusFile source) { return new V1.CorpusFile { TextId = source.TextId, Format = (V1.FileFormat)source.Format, - Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.Filename) + Location = Path.Combine(_dataFileOptions.CurrentValue.FilesDirectory, source.GetFilename()) }; } } diff --git a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs index 516e634e..160c751f 100644 --- a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs +++ b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs @@ -5,11 +5,13 @@ namespace Serval.Translation.Services; public class PretranslationService( IRepository pretranslations, IRepository engines, - IScriptureDataFileService scriptureDataFileService + IScriptureDataFileService scriptureDataFileService, + IRequestClient getDataFileClient ) : EntityServiceBase(pretranslations), IPretranslationService { private readonly IRepository _engines = engines; private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + private readonly IRequestClient _getDataFileClient = getDataFileClient; public async Task> GetAllAsync( string engineId, @@ -40,6 +42,7 @@ public async Task GetUsfmAsync( ) { Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + await engine!.PopulateFilenamesAsync(_getDataFileClient, cancellationToken); Corpus? corpus = engine?.Corpora.SingleOrDefault(c => c.Id == corpusId); ParallelCorpus? parallelCorpus = engine?.ParallelCorpora.SingleOrDefault(c => c.Id == corpusId); @@ -63,10 +66,10 @@ public async Task GetUsfmAsync( throw new InvalidOperationException("USFM format is not valid for non-Scripture corpora."); ParatextProjectSettings sourceSettings = _scriptureDataFileService.GetParatextProjectSettings( - sourceFile.Filename + sourceFile.GetFilename() ); ParatextProjectSettings targetSettings = _scriptureDataFileService.GetParatextProjectSettings( - targetFile.Filename + targetFile.GetFilename() ); IEnumerable<(IReadOnlyList Refs, string Translation)> pretranslations = ( @@ -90,7 +93,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) ((IReadOnlyList)p.Refs.Select(r => r.ToRelaxed()).ToArray(), p.Translation) ); using Shared.Services.ZipParatextProjectTextUpdater updater = - _scriptureDataFileService.GetZipParatextProjectTextUpdater(targetFile.Filename); + _scriptureDataFileService.GetZipParatextProjectTextUpdater(targetFile.GetFilename()); string usfm = ""; switch (textOrigin) { @@ -139,7 +142,7 @@ await GetAllAsync(engineId, modelRevision, corpusId, textId, cancellationToken) if (template is PretranslationUsfmTemplate.Auto or PretranslationUsfmTemplate.Source) { using Shared.Services.ZipParatextProjectTextUpdater updater = - _scriptureDataFileService.GetZipParatextProjectTextUpdater(sourceFile.Filename); + _scriptureDataFileService.GetZipParatextProjectTextUpdater(sourceFile.GetFilename()); // Copy and update the source book if it exists switch (textOrigin) diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs index 328964e2..52058c23 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/AssessmentEngineTests.cs @@ -181,7 +181,6 @@ public async Task AddEngineAsync() { Id = dataFile.Id, Format = Shared.Contracts.FileFormat.Paratext, - Filename = dataFile.Filename, TextId = "all" } ] diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index d66b3557..7402c01c 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -184,21 +184,21 @@ public async Task SetUp() Id = SOURCE_CORPUS_ID_1, Language = "en", Owner = "client1", - Files = [new() { File = srcFile, TextId = "all" }] + Files = [new() { FileReference = Map(srcFile), TextId = "all" }] }; var srcCorpus2 = new DataFiles.Models.Corpus { Id = SOURCE_CORPUS_ID_2, Language = "en", Owner = "client1", - Files = [new() { File = srcFile, TextId = "all" }] + Files = [new() { FileReference = Map(srcFile), TextId = "all" }] }; var trgCorpus = new DataFiles.Models.Corpus { Id = TARGET_CORPUS_ID, Language = "en", Owner = "client1", - Files = [new() { File = trgFile, TextId = "all" }] + Files = [new() { FileReference = Map(trgFile), TextId = "all" }] }; await _env.Corpora.InsertAllAsync([srcCorpus, srcCorpus2, trgCorpus]); } @@ -599,11 +599,6 @@ public async Task AddCorpusToEngineByIdAsync(IEnumerable scope, int expe }); Engine? engine = await _env.Engines.GetAsync(engineId); Assert.That(engine, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(engine.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE1_FILENAME)); - Assert.That(engine.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); - }); break; } case 403: @@ -661,11 +656,6 @@ string engineId await client.UpdateCorpusAsync(engineId, result.Id, updateConfig); Engine? engine = await _env.Engines.GetAsync(engineId); Assert.That(engine, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(engine.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); - Assert.That(engine.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE1_FILENAME)); - }); break; } case 400: @@ -836,11 +826,6 @@ public async Task AddParallelCorpusToEngineByIdAsync() }); Engine? engine = await _env.Engines.GetAsync(ECHO_ENGINE1_ID); Assert.That(engine, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(engine.ParallelCorpora[0].SourceCorpora[0].Files[0].Filename, Is.EqualTo(FILE1_FILENAME)); - Assert.That(engine.ParallelCorpora[0].TargetCorpora[0].Files[0].Filename, Is.EqualTo(FILE2_FILENAME)); - }); } public void AddParallelCorpusToEngineById_NoSuchEngine() @@ -885,11 +870,6 @@ public async Task UpdateParallelCorpusByIdForEngineByIdAsync() await client.UpdateParallelCorpusAsync(ECHO_ENGINE1_ID, result.Id, updateConfig); Engine? engine = await _env.Engines.GetAsync(ECHO_ENGINE1_ID); Assert.That(engine, Is.Not.Null); - Assert.Multiple(() => - { - Assert.That(engine.ParallelCorpora[0].SourceCorpora[0].Files[0].Filename, Is.EqualTo(FILE1_FILENAME)); - Assert.That(engine.ParallelCorpora[0].TargetCorpora[0].Files[0].Filename, Is.EqualTo(FILE2_FILENAME)); - }); } [Test] @@ -2380,6 +2360,14 @@ protected override void DisposeManagedResources() ResetDatabases(); } } + + static DataFiles.Models.DataFileReference Map(DataFiles.Models.DataFile file) => + new DataFiles.Models.DataFileReference + { + Id = file.Id, + Name = file.Name, + Format = file.Format + }; } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs index 22cdd14e..8c88699b 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs +++ b/src/Serval/test/Serval.DataFiles.Tests/Services/CorpusServiceTests.cs @@ -5,13 +5,11 @@ public class CorpusServiceTests { private const string CorpusId = "c00000000000000000000001"; - private static readonly DataFile DefaultDataFile = + private static readonly DataFileReference DefaultDataFile = new() { Id = "df0000000000000000000001", - Owner = "owner1", Name = "file1", - Filename = "file1.txt", Format = FileFormat.Text }; private static readonly Corpus DefaultCorpus = @@ -21,7 +19,7 @@ public class CorpusServiceTests Owner = "owner1", Name = "corpus1", Language = "en", - Files = new List() { new() { File = DefaultDataFile } } + Files = new List() { new() { FileReference = DefaultDataFile } } }; [Test] diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 0da83cf1..9546414e 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -133,7 +133,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -154,7 +154,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -207,7 +207,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -229,7 +229,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -282,7 +282,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -304,7 +304,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -356,7 +356,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -377,7 +377,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -430,7 +430,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -451,7 +451,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -504,7 +504,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -525,7 +525,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -550,7 +550,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -571,7 +571,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -652,7 +652,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -684,7 +684,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -736,7 +736,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -757,7 +757,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -833,7 +833,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -849,7 +849,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -872,7 +872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -888,7 +888,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -965,7 +965,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -988,7 +988,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1065,7 +1065,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1088,7 +1088,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1114,7 +1114,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1136,7 +1136,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1212,7 +1212,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1228,7 +1228,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1251,7 +1251,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1267,7 +1267,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1363,7 +1363,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1379,7 +1379,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1412,7 +1412,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1428,7 +1428,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1495,7 +1495,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1522,7 +1522,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1552,7 +1552,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1579,7 +1579,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1640,7 +1640,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1656,7 +1656,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1675,7 +1675,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1691,7 +1691,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1734,7 +1734,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1750,7 +1750,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1769,7 +1769,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1785,7 +1785,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1845,7 +1845,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1872,7 +1872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1891,7 +1891,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1907,7 +1907,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1944,14 +1944,12 @@ public async Task UpdateCorpusAsync() new() { Id = "file1", - Filename = "file1.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" }, new() { Id = "file3", - Filename = "file3.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text2" }, @@ -2115,7 +2113,8 @@ public TestEnvironment() dataFileOptions, new MemoryDataAccessContext(), new LoggerFactory(), - scriptureDataFileService + scriptureDataFileService, + Substitute.For>() ); } @@ -2144,7 +2143,6 @@ public async Task CreateEngineWithTextFilesAsync() new() { Id = "file1", - Filename = "file1.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2154,7 +2152,6 @@ public async Task CreateEngineWithTextFilesAsync() new() { Id = "file2", - Filename = "file2.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2187,7 +2184,6 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() new() { Id = "file1", - Filename = "file1.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2197,7 +2193,6 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() new() { Id = "file2", - Filename = "file2.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2213,7 +2208,6 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() new() { Id = "file3", - Filename = "file3.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2223,7 +2217,6 @@ public async Task CreateMultipleCorporaEngineWithTextFilesAsync() new() { Id = "file4", - Filename = "file4.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "text1" } @@ -2256,7 +2249,6 @@ public async Task CreateEngineWithParatextProjectAsync() new() { Id = "file1", - Filename = "file1.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file1.zip" } @@ -2266,7 +2258,6 @@ public async Task CreateEngineWithParatextProjectAsync() new() { Id = "file2", - Filename = "file2.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file2.zip" } @@ -2292,7 +2283,7 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2304,7 +2295,6 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "file1", - Filename = "file1.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MAT" } @@ -2320,14 +2310,13 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "file3", - Filename = "file3.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MRK" } ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2339,7 +2328,6 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "file2", - Filename = "file2.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MAT" } @@ -2355,7 +2343,6 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "file4", - Filename = "file4.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MRK" } @@ -2383,7 +2370,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2395,14 +2382,13 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "file1", - Filename = "file1.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MAT" } ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2414,7 +2400,6 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "file2", - Filename = "file2.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MAT" } @@ -2425,7 +2410,7 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus2", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2437,14 +2422,13 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "file3", - Filename = "file3.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MRK" } ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2456,7 +2440,6 @@ public async Task CreateMultipleParallelCorpusEngineWithTextFilesAsync() new() { Id = "file4", - Filename = "file4.txt", Format = Shared.Contracts.FileFormat.Text, TextId = "MRK" } @@ -2484,7 +2467,7 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -2496,7 +2479,6 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "file1", - Filename = "file1.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file1.zip" } @@ -2512,14 +2494,13 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "file3", - Filename = "file3.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file3.zip" } ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -2531,7 +2512,6 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "file2", - Filename = "file2.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file2.zip" } @@ -2547,7 +2527,6 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "file4", - Filename = "file4.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "file4.zip" } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index 5aca4ed6..bc76727e 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -270,22 +270,22 @@ private class TestEnvironment : IDisposable { public TestEnvironment() { - CorpusFile file1 = + Shared.Models.CorpusFile file1 = new() { Id = "file1", - Filename = "file1.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "project1" }; - CorpusFile file2 = + file1.SetFilename("file1.zip"); + Shared.Models.CorpusFile file2 = new() { Id = "file2", - Filename = "file2.zip", Format = Shared.Contracts.FileFormat.Paratext, TextId = "project1" }; + file2.SetFilename("file2.zip"); Engines = new MemoryRepository( [ new() @@ -321,7 +321,7 @@ public TestEnvironment() new() { Id = "parallel_corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -330,7 +330,7 @@ public TestEnvironment() Files = [file1], } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -392,6 +392,7 @@ public TestEnvironment() ScriptureDataFileService = Substitute.For(); ScriptureDataFileService.GetParatextProjectSettings("file1.zip").Returns(CreateProjectSettings("SRC")); ScriptureDataFileService.GetParatextProjectSettings("file2.zip").Returns(CreateProjectSettings("TRG")); + GetDataFileClient = Substitute.For>(); var zipSubstituteSource = Substitute.For(); var zipSubstituteTarget = Substitute.For(); zipSubstituteSource @@ -425,13 +426,15 @@ Shared.Services.ZipParatextProjectTextUpdater GetTextUpdater(string type) } ScriptureDataFileService.GetZipParatextProjectTextUpdater("file1.zip").Returns(x => GetTextUpdater("SRC")); ScriptureDataFileService.GetZipParatextProjectTextUpdater("file2.zip").Returns(x => GetTextUpdater("TRG")); - Service = new PretranslationService(Pretranslations, Engines, ScriptureDataFileService); + + Service = new PretranslationService(Pretranslations, Engines, ScriptureDataFileService, GetDataFileClient); } public PretranslationService Service { get; } public MemoryRepository Pretranslations { get; } public MemoryRepository Engines { get; } public IScriptureDataFileService ScriptureDataFileService { get; } + public IRequestClient GetDataFileClient { get; } public IZipContainer TargetZipContainer { get; } public IList TextUpdaters { get; } diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs index ef8a3ff7..9116c0c9 100644 --- a/src/Serval/test/Serval.Translation.Tests/Usings.cs +++ b/src/Serval/test/Serval.Translation.Tests/Usings.cs @@ -7,6 +7,7 @@ global using NSubstitute; global using NUnit.Framework; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Services; global using Serval.Shared.Utils; global using Serval.Translation.Contracts;