Build summary gprc

src/Machine/src/Serval.Machine.Shared/Models/Build.cs

@@ -29,4 +29,5 @@ public record Build
     public required BuildJobRunnerType BuildJobRunner { get; init; }
     public required BuildStage Stage { get; init; }
     public string? Options { get; set; }
+    public Dictionary<string, int>? Statistics { get; set; }
 }

src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs

@@ -27,4 +27,11 @@ Task InsertPretranslationsAsync(
         Stream pretranslationsStream,
         CancellationToken cancellationToken = default
     );
+
+    Task UpdateBuildStatisticsAsync(
+        string engineId,
+        string buildId,
+        IDictionary<string, string> statistics,
+        CancellationToken cancellationToken = default
+    );
 }

src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs

@@ -33,6 +33,8 @@ CancellationToken cancellationToken
             await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken);
         }
 
+        // here's where to add more data to send
+
         int additionalCorpusSize = await SaveModelAsync(engineId, buildId);
         await DataAccessContext.WithTransactionAsync(
             async (ct) =>

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -82,6 +82,13 @@ CancellationToken cancellationToken
             );
         }
 
+        var statistics = new Dictionary<string, string>()
+        {
+            { "initialTrainCount", trainCount.ToString(CultureInfo.InvariantCulture) },
+            { "initialPretranslateCount", pretranslateCount.ToString(CultureInfo.InvariantCulture) }
+        };
+        await PlatformService.UpdateBuildStatisticsAsync(engineId, buildId, statistics, cancellationToken);
+
         cancellationToken.ThrowIfCancellationRequested();
 
         bool canceling = !await BuildJobService.StartBuildJobAsync(

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs

@@ -11,4 +11,5 @@ public static class ServalPlatformOutboxConstants
     public const string BuildRestarting = "BuildRestarting";
     public const string InsertPretranslations = "InsertPretranslations";
     public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize";
+    public const string UpdateBuildStatistics = "UpdateBuildStatistics";
 }

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs

@@ -85,6 +85,12 @@ await _client.IncrementTranslationEngineCorpusSizeAsync(
                     cancellationToken: cancellationToken
                 );
                 break;
+            case ServalPlatformOutboxConstants.UpdateBuildStatistics:
+                await _client.UpdateBuildStatisticsAsync(
+                    JsonSerializer.Deserialize<UpdateBuildStatisticsRequest>(content!),
+                    cancellationToken: cancellationToken
+                );
+                break;
             default:
                 throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'.");
         }

src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs

@@ -137,4 +137,22 @@ await _outboxService.EnqueueMessageAsync(
             cancellationToken: cancellationToken
         );
     }
+
+    public async Task UpdateBuildStatisticsAsync(
+        string engineId,
+        string buildId,
+        IDictionary<string, string> statistics,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var request = new UpdateBuildStatisticsRequest { EngineId = engineId, BuildId = buildId };
+        request.Statistics.Add(statistics);
+        await _outboxService.EnqueueMessageAsync(
+            ServalPlatformOutboxConstants.OutboxId,
+            ServalPlatformOutboxConstants.UpdateBuildStatistics,
+            engineId,
+            JsonSerializer.Serialize(request),
+            cancellationToken: cancellationToken
+        );
+    }
 }

src/Serval/src/Serval.Client/Client.g.cs

@@ -4541,6 +4541,16 @@ public partial interface ITranslationEnginesClient
         /// <exception cref="ServalApiException">A server side error occurred.</exception>
         System.Threading.Tasks.Task<ModelDownloadUrl> GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));
 
+        /// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
+        /// <summary>
+        /// Get a build summary for a given build ID.
+        /// </summary>
+        /// <param name="id">The translation engine id</param>
+        /// <param name="buildId">The build job id</param>
+        /// <returns>The build summary</returns>
+        /// <exception cref="ServalApiException">A server side error occurred.</exception>
+        System.Threading.Tasks.Task<TranslationBuildSummary> GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));
+
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
@@ -7882,6 +7892,116 @@ public string BaseUrl
             }
         }
 
+        /// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
+        /// <summary>
+        /// Get a build summary for a given build ID.
+        /// </summary>
+        /// <param name="id">The translation engine id</param>
+        /// <param name="buildId">The build job id</param>
+        /// <returns>The build summary</returns>
+        /// <exception cref="ServalApiException">A server side error occurred.</exception>
+        public virtual async System.Threading.Tasks.Task<TranslationBuildSummary> GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken))
+        {
+            if (id == null)
+                throw new System.ArgumentNullException("id");
+
+            if (buildId == null)
+                throw new System.ArgumentNullException("buildId");
+
+            var client_ = _httpClient;
+            var disposeClient_ = false;
+            try
+            {
+                using (var request_ = new System.Net.Http.HttpRequestMessage())
+                {
+                    request_.Method = new System.Net.Http.HttpMethod("GET");
+                    request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json"));
+
+                    var urlBuilder_ = new System.Text.StringBuilder();
+                    if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl);
+                    // Operation Path: "translation/engines/{id}/builds/{buildId}/summary"
+                    urlBuilder_.Append("translation/engines/");
+                    urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture)));
+                    urlBuilder_.Append("/builds/");
+                    urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture)));
+                    urlBuilder_.Append("/summary");
+
+                    PrepareRequest(client_, request_, urlBuilder_);
+
+                    var url_ = urlBuilder_.ToString();
+                    request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute);
+
+                    PrepareRequest(client_, request_, url_);
+
+                    var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
+                    var disposeResponse_ = true;
+                    try
+                    {
+                        var headers_ = new System.Collections.Generic.Dictionary<string, System.Collections.Generic.IEnumerable<string>>();
+                        foreach (var item_ in response_.Headers)
+                            headers_[item_.Key] = item_.Value;
+                        if (response_.Content != null && response_.Content.Headers != null)
+                        {
+                            foreach (var item_ in response_.Content.Headers)
+                                headers_[item_.Key] = item_.Value;
+                        }
+
+                        ProcessResponse(client_, response_);
+
+                        var status_ = (int)response_.StatusCode;
+                        if (status_ == 200)
+                        {
+                            var objectResponse_ = await ReadObjectResponseAsync<TranslationBuildSummary>(response_, headers_, cancellationToken).ConfigureAwait(false);
+                            if (objectResponse_.Object == null)
+                            {
+                                throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null);
+                            }
+                            return objectResponse_.Object;
+                        }
+                        else
+                        if (status_ == 401)
+                        {
+                            string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null);
+                        }
+                        else
+                        if (status_ == 403)
+                        {
+                            string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null);
+                        }
+                        else
+                        if (status_ == 404)
+                        {
+                            string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null);
+                        }
+                        else
+                        if (status_ == 503)
+                        {
+                            string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null);
+                        }
+                        else
+                        {
+                            var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null);
+                        }
+                    }
+                    finally
+                    {
+                        if (disposeResponse_)
+                            response_.Dispose();
+                    }
+                }
+            }
+            finally
+            {
+                if (disposeClient_)
+                    client_.Dispose();
+            }
+        }
+
         protected struct ObjectResponseResult<T>
         {
             public ObjectResponseResult(T responseObject, string responseText)
@@ -9849,6 +9969,8 @@ public partial class TranslationBuild
         [Newtonsoft.Json.JsonProperty("deploymentVersion", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string? DeploymentVersion { get; set; } = default!;
 
+        [Newtonsoft.Json.JsonProperty("statistics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, string>? Statistics { get; set; } = default!;
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
@@ -10012,6 +10134,21 @@ public partial class ModelDownloadUrl
 
     }
 
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class TranslationBuildSummary
+    {
+        [Newtonsoft.Json.JsonProperty("linesTrainedOn", Required = Newtonsoft.Json.Required.Always)]
+        public int LinesTrainedOn { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("linesPretranslated", Required = Newtonsoft.Json.Required.Always)]
+        public int LinesPretranslated { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("totalBuildTime", Required = Newtonsoft.Json.Required.Always)]
+        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
+        public System.TimeSpan TotalBuildTime { get; set; } = default!;
+
+    }
+
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class Queue
     {

src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto

@@ -14,6 +14,7 @@ service TranslationPlatformApi {
 
     rpc IncrementTranslationEngineCorpusSize(IncrementTranslationEngineCorpusSizeRequest) returns (google.protobuf.Empty);
     rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty);
+    rpc UpdateBuildStatistics(UpdateBuildStatisticsRequest) returns (google.protobuf.Empty);
 }
 
 message UpdateBuildStatusRequest {
@@ -59,3 +60,9 @@ message InsertPretranslationsRequest {
     repeated string refs = 4;
     string translation = 5;
 }
+
+message UpdateBuildStatisticsRequest {
+    string engine_id = 1;
+    string build_id = 2;
+    map<string, string> Statistics = 3;
+}

src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs

@@ -28,4 +28,5 @@ public record TranslationBuildDto
     /// </example>
     public object? Options { get; init; }
     public string? DeploymentVersion { get; init; }
+    public Dictionary<string, string>? Statistics { get; set; }
 }

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1195,6 +1195,79 @@ CancellationToken cancellationToken
         return Ok(Map(modelInfo));
     }
 
+    /// <summary>
+    /// Get a build summary for a given build ID.
+    /// </summary>
+    /// <param name="id">The translation engine id</param>
+    /// <param name="buildId">The build job id</param>
+    /// <param name="cancellationToken"></param>
+    /// <response code="200">The build summary</response>
+    /// <response code="401">The client is not authenticated.</response>
+    /// <response code="403">The authenticated client does not own the translation engine.</response>
+    /// <response code="404">The engine or build does not exist.</response>
+    /// <response code="503">A necessary service is currently unavailable. Check `/health` for more details.</response>
+    [Authorize(Scopes.ReadTranslationEngines)]
+    [HttpGet("{id}/builds/{buildId}/summary")]
+    [ProducesResponseType(StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)]
+    public async Task<ActionResult<TranslationBuildSummary>> GetBuildSummaryAsync(
+        [NotNull] string id,
+        [NotNull] string buildId,
+        CancellationToken cancellationToken
+    )
+    {
+        await AuthorizeAsync(id, cancellationToken);
+
+        Build build = await _buildService.GetAsync(buildId, cancellationToken);
+
+        // Calculate total time for build
+        TimeSpan totalBuildTime = build.DateFinished.HasValue
+            ? build.DateFinished.Value - build.DateStarted
+            : TimeSpan.Zero;
+
+        var summary = new TranslationBuildSummary
+        {
+            LinesTrainedOn = 1,
+            LinesPretranslated = 2,
+            TotalBuildTime = totalBuildTime
+        };
+
+        return Ok(summary);
+    }
+
+    // private async Task<(int TrainCount, int PretranslateCount)> GetBuildCountsFromLogAsync(
+    //     string buildId,
+    //     CancellationToken cancellationToken
+    // )
+    // {
+    //     // 1. Access the logs for the given buildId.
+    //     //    - This depends on how you store and access your logs (e.g., file system, database, logging service).
+    //     //    - Replace this with your actual log access logic.
+    //     string logContent = await GetLogContentAsync(buildId, cancellationToken);
+
+    //     // 2. Parse the log content to find the relevant log entry.
+    //     //    - This assumes the log entry format you provided in the PreprocessBuildJob.
+    //     //    - Adjust the parsing logic if your log format is different.
+    //     int trainCount = 0;
+    //     int pretranslateCount = 0;
+
+    //     // Example parsing using regular expressions (adjust as needed):
+    //     Match match = Regex.Match(
+    //         logContent,
+    //         @"{""Event"",""BuildPreprocess""},{""EngineId"",""\w+""},{""BuildId"",""\w+""},{""NumTrainRows"",(\d+)},{""NumPretranslateRows"",(\d+)},"
+    //     );
+    //     if (match.Success)
+    //     {
+    //         trainCount = int.Parse(match.Groups[1].Value);
+    //         pretranslateCount = int.Parse(match.Groups[2].Value);
+    //     }
+
+    //     return (trainCount, pretranslateCount);
+    // }
+
     private async Task AuthorizeAsync(string id, CancellationToken cancellationToken)
     {
         Engine engine = await _engineService.GetAsync(id, cancellationToken);

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -13,7 +13,10 @@ public record Build : IEntity
     public string? Message { get; init; }
     public int? QueueDepth { get; init; }
     public JobState State { get; init; } = JobState.Pending;
+    public DateTime DateStarted { get; init; } = DateTime.UtcNow;
     public DateTime? DateFinished { get; init; }
     public IReadOnlyDictionary<string, object>? Options { get; init; }
     public string? DeploymentVersion { get; init; }
+    public Dictionary<string, string> Statistics { get; init; } =
+        new Dictionary<string, string>() { { "trainCount", "0" } };
 }

src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs

@@ -0,0 +1,8 @@
+namespace Serval.Translation.Models;
+
+public class TranslationBuildSummary
+{
+    public int LinesTrainedOn { get; set; }
+    public int LinesPretranslated { get; set; }
+    public TimeSpan TotalBuildTime { get; set; }
+}