When no filter is specified, only include kbts that are associated wi…

…th the texts of the corpus
sillsdev · Nov 27, 2024 · 788beb9 · 788beb9
1 parent 069dc40
commit 788beb9
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 13 deletions.
diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
@@ -118,10 +118,10 @@ public async Task RunAsync_EnableKeyTerms()
         (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
         Assert.Multiple(() =>
         {
-            Assert.That(src1Count, Is.EqualTo(0));
+            Assert.That(src1Count, Is.EqualTo(14));
             Assert.That(src2Count, Is.EqualTo(0));
-            Assert.That(trgCount, Is.EqualTo(0));
-            Assert.That(termCount, Is.EqualTo(5726));
+            Assert.That(trgCount, Is.EqualTo(1));
+            Assert.That(termCount, Is.EqualTo(166));
         });
     }
 
@@ -136,9 +136,9 @@ public async Task RunAsync_DisableKeyTerms()
         (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
         Assert.Multiple(() =>
         {
-            Assert.That(src1Count, Is.EqualTo(0));
+            Assert.That(src1Count, Is.EqualTo(14));
             Assert.That(src2Count, Is.EqualTo(0));
-            Assert.That(trgCount, Is.EqualTo(0));
+            Assert.That(trgCount, Is.EqualTo(1));
             Assert.That(termCount, Is.EqualTo(0));
         });
     }
@@ -853,8 +853,8 @@ public TestEnvironment()
                         Id = "src_1",
                         Language = "es",
                         Files = [ParatextFile("pt-source1")],
-                        TrainOnTextIds = [],
-                        PretranslateTextIds = []
+                        TrainOnTextIds = null,
+                        PretranslateTextIds = null
                     }
                 },
                 TargetCorpora = new List<MonolingualCorpus>()
@@ -864,7 +864,7 @@ public TestEnvironment()
                         Id = "trg_1",
                         Language = "en",
                         Files = [ParatextFile("pt-target1")],
-                        TrainOnTextIds = []
+                        TrainOnTextIds = null
                     }
                 }
             };

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs
@@ -40,7 +40,7 @@ public IEnumerable<ITextCorpus> CreateTermCorpora(
         IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
     )
     {
-        foreach ((CorpusFile file, Dictionary<string, HashSet<int>> chapters) in corpora)
+        foreach ((CorpusFile file, Dictionary<string, HashSet<int>>? chapters) in corpora)
         {
             switch (file.Format)
             {

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs
@@ -83,10 +83,18 @@ public void Preprocess(
             if (useKeyTerms)
             {
                 ITextCorpus[]? sourceTermCorpora = _corpusService
-                    .CreateTermCorpora(corpus.SourceCorpora.SelectMany(GetChaptersPerFile).ToArray())
+                    .CreateTermCorpora(
+                        sourceCorpora
+                            .SelectMany(corpus => GetChaptersPerFile(corpus.Corpus, corpus.TextCorpus))
+                            .ToArray()
+                    )
                     .ToArray();
                 ITextCorpus[]? targetTermCorpora = _corpusService
-                    .CreateTermCorpora(corpus.TargetCorpora.SelectMany(GetChaptersPerFile).ToArray())
+                    .CreateTermCorpora(
+                        targetCorpora
+                            .SelectMany(corpus => GetChaptersPerFile(corpus.Corpus, corpus.TextCorpus))
+                            .ToArray()
+                    )
                     .ToArray();
                 if (sourceTermCorpora is not null && targetTermCorpora is not null)
                 {
@@ -114,15 +122,16 @@ public void Preprocess(
     }
 
     private static IEnumerable<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> GetChaptersPerFile(
-        MonolingualCorpus mc
+        MonolingualCorpus mc,
+        ITextCorpus tc
     )
     {
         Dictionary<string, HashSet<int>>? chapters = mc.TrainOnChapters;
         if (chapters is null && mc.TrainOnTextIds is not null)
         {
             chapters = mc.TrainOnTextIds.Select(tid => (tid, new HashSet<int> { })).ToDictionary();
         }
-        chapters ??= [];
+        chapters ??= tc.Texts.Select(t => (t.Id, new HashSet<int>() { })).ToDictionary();
         return mc.Files.Select(f => (f, chapters));
     }