Skip to content

Commit

Permalink
Use chapter-filtering for terms (#545)
Browse files Browse the repository at this point in the history 
Use chapter-filtering for terms
  • Loading branch information
@Enkidu93
Enkidu93 authored Nov 27, 2024
1 parent 0635292 commit 03e4716
Show file tree
Hide file tree
Showing 9 changed files with 308 additions and 26 deletions.
6 changes: 3 additions & 3 deletions src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
<PackageReference Include="Hangfire.Mongo" Version="1.10.8" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.8" />
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
Expand Down
258 changes: 249 additions & 9 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,10 @@ public async Task RunAsync_EnableKeyTerms()
(int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
Assert.Multiple(() =>
{
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src1Count, Is.EqualTo(14));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(termCount, Is.EqualTo(5726));
Assert.That(trgCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(166));
});
}

Expand All @@ -136,9 +136,9 @@ public async Task RunAsync_DisableKeyTerms()
(int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
Assert.Multiple(() =>
{
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src1Count, Is.EqualTo(14));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(0));
});
}
Expand Down Expand Up @@ -359,7 +359,7 @@ public void RunAsync_OnlyParseSelectedBooks_PretranslateOnBadBook()
}

[Test]
public async Task ParallelCorpusLogic()
public async Task ParallelCorpusAsync()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
Expand Down Expand Up @@ -508,6 +508,246 @@ public async Task ParallelCorpusLogic()
});
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"LEV",
new() { }
}
},
PretranslateChapters = new()
{
{
"1CH",
new() { }
}
}
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
},
PretranslateChapters = new() { }
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
}
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
},
{
"LEV",
new() { }
}
}
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.
Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms_TextIds()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnTextIds = ["MAT", "LEV"],
PretranslateTextIds = ["1CH"]
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnTextIds = ["MAT", "MRK"],
PretranslateTextIds = []
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnTextIds = ["MAT", "MRK"]
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnTextIds = ["MAT", "MRK", "LEV"]
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source one, chapter two, verse one.
Source one, chapter two, verse two.
Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.
Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
Target one, chapter two, verse one.
Target one, chapter two, verse three.
",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

private class TestEnvironment : DisposableBase
{
private static readonly string TestDataPath = Path.Combine(
Expand Down Expand Up @@ -613,8 +853,8 @@ public TestEnvironment()
Id = "src_1",
Language = "es",
Files = [ParatextFile("pt-source1")],
TrainOnTextIds = [],
PretranslateTextIds = []
TrainOnTextIds = null,
PretranslateTextIds = null
}
},
TargetCorpora = new List<MonolingualCorpus>()
Expand All @@ -624,7 +864,7 @@ public TestEnvironment()
Id = "trg_1",
Language = "en",
Files = [ParatextFile("pt-target1")],
TrainOnTextIds = []
TrainOnTextIds = null
}
}
};
Expand Down
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
2 changes: 1 addition & 1 deletion src/Serval/src/Serval.Shared/Serval.Shared.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<PackageReference Include="Grpc.Core.Api" Version="2.65.0" />
<PackageReference Include="Grpc.HealthCheck" Version="2.65.0" />
<PackageReference Include="Grpc.Net.ClientFactory" Version="2.65.0" />
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
<PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
<PackageReference Include="SIL.Scripture" Version="12.0.1"/>
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
</ItemGroup>

<ItemGroup>
Expand Down
8 changes: 5 additions & 3 deletions src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@ public IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> file
return corpora;
}

public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files)
public IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
)
{
foreach (CorpusFile file in files)
foreach ((CorpusFile file, Dictionary<string, HashSet<int>> chapters) in corpora)
{
switch (file.Format)
{
case FileFormat.Paratext:
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], chapters: chapters);
break;
}
}
Expand Down
4 changes: 3 additions & 1 deletion src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ICorpusService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@
public interface ICorpusService
{
IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
);
}
Loading

0 comments on commit 03e4716

Please sign in to comment.