Programming exercises: Show build logs for submission results (#8170)

src/main/java/de/tum/in/www1/artemis/service/BuildLogEntryService.java

@@ -2,13 +2,25 @@
 
 import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Profile;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 import de.tum.in.www1.artemis.domain.BuildLogEntry;
@@ -22,10 +34,15 @@
 @Service
 public class BuildLogEntryService {
 
+    private static final Logger log = LoggerFactory.getLogger(BuildLogEntryService.class);
+
     private final BuildLogEntryRepository buildLogEntryRepository;
 
     private final ProgrammingSubmissionRepository programmingSubmissionRepository;
 
+    @Value("${artemis.continuous-integration.build-log.file-expiry-days:30}")
+    private int expiryDays;
+
     public BuildLogEntryService(BuildLogEntryRepository buildLogEntryRepository, ProgrammingSubmissionRepository programmingSubmissionRepository) {
         this.buildLogEntryRepository = buildLogEntryRepository;
         this.programmingSubmissionRepository = programmingSubmissionRepository;
@@ -256,4 +273,90 @@ public void deleteBuildLogEntriesForProgrammingSubmission(ProgrammingSubmission
         programmingSubmissionRepository.save(programmingSubmission);
         buildLogEntryRepository.deleteByProgrammingSubmissionId(programmingSubmission.getId());
     }
+
+    /**
+     * Save the build logs for a given submission to a file
+     *
+     * @param buildLogEntries the build logs to save
+     * @param resultId        the id of the result for which to save the build logs
+     */
+    public void saveBuildLogsToFile(List<BuildLogEntry> buildLogEntries, String resultId) {
+
+        Path buildLogsPath = Path.of("buildLogs");
+
+        if (!Files.exists(buildLogsPath)) {
+            try {
+                Files.createDirectory(buildLogsPath);
+            }
+            catch (Exception e) {
+                throw new IllegalStateException("Could not create directory for build logs", e);
+            }
+        }
+
+        Path logPath = buildLogsPath.resolve(resultId + ".log");
+
+        StringBuilder logsStringBuilder = new StringBuilder();
+        for (BuildLogEntry buildLogEntry : buildLogEntries) {
+            logsStringBuilder.append(buildLogEntry.getTime()).append("\t").append(buildLogEntry.getLog());
+        }
+
+        try {
+            FileUtils.writeStringToFile(logPath.toFile(), logsStringBuilder.toString(), StandardCharsets.UTF_8);
+            log.debug("Saved build logs for result {} to file {}", resultId, logPath);
+        }
+        catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Retrieves the build logs for a given submission from a file.
+     *
+     * @param resultId the id of the result for which to retrieve the build logs
+     * @return the build logs as a string or null if the file could not be found (e.g. if the build logs have been deleted)
+     */
+    public FileSystemResource retrieveBuildLogsFromFileForResult(String resultId) {
+        Path buildLogsPath = Path.of("buildLogs");
+        Path logPath = buildLogsPath.resolve(resultId + ".log");
+
+        FileSystemResource fileSystemResource = new FileSystemResource(logPath);
+        if (fileSystemResource.exists()) {
+            log.debug("Retrieved build logs for result {} from file {}", resultId, logPath);
+            return fileSystemResource;
+        }
+        else {
+            log.warn("Could not find build logs for result {} in file {}", resultId, logPath);
+            return null;
+        }
+    }
+
+    /**
+     * Deletes all build log files that are older than {@link #expiryDays} days on a schedule
+     */
+    @Scheduled(cron = "${artemis.continuous-integration.build-log.cleanup-schedule:0 0 3 1 * ?}")
+    public void deleteOldBuildLogsFiles() {
+        log.info("Deleting old build log files");
+        ZonedDateTime now = ZonedDateTime.now();
+        Path buildLogsPath = Path.of("buildLogs");
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(buildLogsPath)) {
+            for (Path file : stream) {
+                ZonedDateTime lastModified = ZonedDateTime.ofInstant(Files.getLastModifiedTime(file).toInstant(), now.getZone());
+                if (lastModified.isBefore(now.minusDays(expiryDays))) {
+                    Files.deleteIfExists(file);
+                    log.info("Deleted old build log file {}", file);
+                }
+            }
+        }
+        catch (IOException e) {
+            log.error("Error occurred while trying to delete old build log files", e);
+        }
+    }
+
+    public boolean resultHasLogFile(String resultId) {
+        Path buildLogsPath = Path.of("buildLogs");
+        Path logPath = buildLogsPath.resolve(resultId + ".log");
+        return Files.exists(logPath);
+    }
+
 }

src/main/java/de/tum/in/www1/artemis/service/ResultService.java

@@ -65,13 +65,16 @@ public class ResultService {
 
     private final LongFeedbackTextRepository longFeedbackTextRepository;
 
+    private final BuildLogEntryService buildLogEntryService;
+
     public ResultService(UserRepository userRepository, ResultRepository resultRepository, Optional<LtiNewResultService> ltiNewResultService,
             ResultWebsocketService resultWebsocketService, ComplaintResponseRepository complaintResponseRepository, RatingRepository ratingRepository,
             FeedbackRepository feedbackRepository, LongFeedbackTextRepository longFeedbackTextRepository, ComplaintRepository complaintRepository,
             ParticipantScoreRepository participantScoreRepository, AuthorizationCheckService authCheckService, ExerciseDateService exerciseDateService,
             TemplateProgrammingExerciseParticipationRepository templateProgrammingExerciseParticipationRepository,
             SolutionProgrammingExerciseParticipationRepository solutionProgrammingExerciseParticipationRepository,
-            ProgrammingExerciseStudentParticipationRepository programmingExerciseStudentParticipationRepository, StudentExamRepository studentExamRepository) {
+            ProgrammingExerciseStudentParticipationRepository programmingExerciseStudentParticipationRepository, StudentExamRepository studentExamRepository,
+            BuildLogEntryService buildLogEntryService) {
         this.userRepository = userRepository;
         this.resultRepository = resultRepository;
         this.ltiNewResultService = ltiNewResultService;
@@ -88,6 +91,7 @@ public ResultService(UserRepository userRepository, ResultRepository resultRepos
         this.solutionProgrammingExerciseParticipationRepository = solutionProgrammingExerciseParticipationRepository;
         this.programmingExerciseStudentParticipationRepository = programmingExerciseStudentParticipationRepository;
         this.studentExamRepository = studentExamRepository;
+        this.buildLogEntryService = buildLogEntryService;
     }
 
     /**
@@ -402,6 +406,25 @@ public Result getResultForParticipationAndCheckAccess(Long participationId, Long
         return result;
     }
 
+    /**
+     * Get a map of result ids to their availability of build log files.
+     *
+     * @param results the results for which to check the availability of build logs
+     * @return a map of result ids to their availability of build log files
+     */
+    public Map<Long, Boolean> getLogsAvailabilityForResults(List<Result> results) {
+        Map<Long, Boolean> logsAvailability = new HashMap<>();
+        for (Result result : results) {
+            if (buildLogEntryService.resultHasLogFile(result.getId().toString())) {
+                logsAvailability.put(result.getId(), true);
+            }
+            else {
+                logsAvailability.put(result.getId(), false);
+            }
+        }
+        return logsAvailability;
+    }
+
     @NotNull
     private List<Feedback> saveFeedbackWithHibernateWorkaround(@NotNull Result result, List<Feedback> feedbackList) {
         // Avoid hibernate exception

src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIResultProcessingService.java

@@ -122,7 +122,8 @@ public void processResult() {
                 if (participation.getProgrammingExercise() == null) {
                     participation.setProgrammingExercise(programmingExerciseRepository.findByParticipationIdOrElseThrow(participation.getId()));
                 }
-                Result result = programmingExerciseGradingService.processNewProgrammingExerciseResult(participation, resultQueueItem.buildResult());
+                Result result = programmingExerciseGradingService.processNewProgrammingExerciseResult(participation, buildResult);
+
                 if (result != null) {
                     programmingMessagingService.notifyUserAboutNewResult(result, participation);
                     addResultToBuildAgentsRecentBuildJobs(buildJob, result);

src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseGradingService.java

@@ -167,7 +167,7 @@ public Result processNewProgrammingExerciseResult(@NotNull ProgrammingExercisePa
             newResult.setSubmission(latestSubmission);
             newResult.setRatedIfNotAfterDueDate();
             // NOTE: the result is not saved yet, but is connected to the submission, the submission is not completely saved yet
-            return processNewProgrammingExerciseResult(participation, newResult);
+            return processNewProgrammingExerciseResult(participation, newResult, buildResult.extractBuildLogs());
         }
         catch (ContinuousIntegrationException ex) {
             log.error("Result for participation {} could not be created", participation.getId(), ex);
@@ -255,7 +255,7 @@ protected ProgrammingSubmission createAndSaveFallbackSubmission(ProgrammingExerc
      * @param newResult     that contains the build result with its feedbacks.
      * @return the result after processing and persisting.
      */
-    private Result processNewProgrammingExerciseResult(final ProgrammingExerciseParticipation participation, final Result newResult) {
+    private Result processNewProgrammingExerciseResult(final ProgrammingExerciseParticipation participation, final Result newResult, final List<BuildLogEntry> buildLogs) {
         ProgrammingExercise programmingExercise = participation.getProgrammingExercise();
         boolean isSolutionParticipation = participation instanceof SolutionProgrammingExerciseParticipation;
         boolean isTemplateParticipation = participation instanceof TemplateProgrammingExerciseParticipation;
@@ -284,7 +284,12 @@ private Result processNewProgrammingExerciseResult(final ProgrammingExercisePart
                 // Adding back dropped submission
                 updatedLatestSemiAutomaticResult.setSubmission(programmingSubmission);
                 programmingSubmissionRepository.save(programmingSubmission);
-                resultRepository.save(updatedLatestSemiAutomaticResult);
+                Result result = resultRepository.save(updatedLatestSemiAutomaticResult);
+
+                // Save the build logs to the file system
+                if (buildLogs != null && !buildLogs.isEmpty()) {
+                    buildLogService.saveBuildLogsToFile(buildLogs, result.getId().toString());
+                }
 
                 return updatedLatestSemiAutomaticResult;
             }
@@ -303,6 +308,10 @@ private Result processNewProgrammingExerciseResult(final ProgrammingExercisePart
         programmingSubmission.addResult(processedResult);
         programmingSubmissionRepository.save(programmingSubmission);
 
+        if (buildLogs != null && !buildLogs.isEmpty()) {
+            buildLogService.saveBuildLogsToFile(buildLogs, processedResult.getId().toString());
+        }
+
         return processedResult;
     }
 

src/main/java/de/tum/in/www1/artemis/web/rest/ResultResource.java

@@ -6,6 +6,7 @@
 import java.net.URISyntaxException;
 import java.time.ZonedDateTime;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
@@ -154,6 +155,27 @@ public ResponseEntity<List<Feedback>> getResultDetails(@PathVariable Long partic
         return new ResponseEntity<>(resultService.filterFeedbackForClient(result), HttpStatus.OK);
     }
 
+    /**
+     * GET /participations/:participationId/results/logs-available : get the logs availability for the results of a participation.
+     *
+     * @param participationId the id of the participation to the results
+     * @return the ResponseEntity with status 200 (OK) and with body the map of resultId and log availability, status 404 (Not Found) if the participation does not exist or 403
+     *         (forbidden) if the user does not have permissions to access the participation.
+     */
+    @GetMapping("participations/{participationId}/results/logs-available")
+    @EnforceAtLeastTutor
+    public ResponseEntity<Map<Long, Boolean>> getLogsAvailabilityForResultsOfParticipation(@PathVariable long participationId) {
+        log.debug("REST request to get logs availability for results of participation : {}", participationId);
+        Participation participation = participationRepository.findByIdElseThrow(participationId);
+        List<Result> results = resultRepository.findAllByParticipationIdOrderByCompletionDateDesc(participationId);
+
+        Map<Long, Boolean> logsAvailable = resultService.getLogsAvailabilityForResults(results);
+
+        participationAuthCheckService.checkCanAccessParticipationElseThrow(participation);
+
+        return new ResponseEntity<>(logsAvailable, HttpStatus.OK);
+    }
+
     /**
      * DELETE /participations/:participationId/results/:resultId : delete the "id" result.
      *

src/main/java/de/tum/in/www1/artemis/web/rest/localci/BuildLogResource.java

@@ -0,0 +1,52 @@
+package de.tum.in.www1.artemis.web.rest.localci;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.core.io.Resource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastEditor;
+import de.tum.in.www1.artemis.service.BuildLogEntryService;
+
+@Profile("localci")
+@RestController
+@RequestMapping("/api")
+public class BuildLogResource {
+
+    private static final Logger log = LoggerFactory.getLogger(BuildLogResource.class);
+
+    private final BuildLogEntryService buildLogEntryService;
+
+    public BuildLogResource(BuildLogEntryService buildLogEntryService) {
+        this.buildLogEntryService = buildLogEntryService;
+    }
+
+    /**
+     * GET /build-log/{resultId} : get the build log for a given result
+     *
+     * @param resultId the id of the result for which to retrieve the build log
+     * @return the ResponseEntity with status 200 (OK) and the build log in the body, or with status 404 (Not Found) if the build log could not be found
+     */
+    @GetMapping("/build-log/{resultId}")
+    @EnforceAtLeastEditor
+    public ResponseEntity<Resource> getBuildLogForSubmission(@PathVariable long resultId) {
+        log.debug("REST request to get the build log for result {}", resultId);
+        HttpHeaders responseHeaders = new HttpHeaders();
+        FileSystemResource buildLog = buildLogEntryService.retrieveBuildLogsFromFileForResult(String.valueOf(resultId));
+        if (buildLog == null) {
+            return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+        }
+        responseHeaders.setContentType(MediaType.TEXT_PLAIN);
+        responseHeaders.setContentDispositionFormData("attachment", "build-" + resultId + ".log");
+        return new ResponseEntity<>(buildLog, responseHeaders, HttpStatus.OK);
+    }
+}

src/main/resources/config/application-artemis.yml

@@ -87,6 +87,9 @@ artemis:
         artemis-authentication-token-value: <token>
         build-timeout: 30 # Does cancel jenkins builds after 30 minutes to remove build that get stuck
         notification-plugin: "ls1tum/artemis-notification-plugin:1.0.0" # Docker image for the generic notification plugin. This value is set in an CI variable in GitLab CI.
+        build-log:
+          file-expiry-days: 30 # The amount of days until build log files can be deleted
+          cleanup-schedule: 0 0 3 1 * ? # Cron expression for schedule to delete old build log files
     git:
         name: Artemis
         email: artemis@xcit.tum.de

src/main/webapp/app/entities/result.model.ts

@@ -23,6 +23,7 @@ export class Result implements BaseEntity {
     public testCaseCount?: number;
     public passedTestCaseCount?: number;
     public codeIssueCount?: number;
+    public logsAvailable?: boolean;
 
     public submission?: Submission;
     public assessor?: User;

src/main/webapp/app/exercises/shared/participation-submission/participation-submission.component.html

@@ -99,6 +99,9 @@ <h4>
                                         <fa-icon [icon]="faTrash" />
                                         {{ 'artemisApp.result.delete.buttonText' | artemisTranslate }}
                                     </span>
+                                    @if (result?.logsAvailable) {
+                                        <a class="detail-link" (click)="viewBuildLogs(result.id)" jhiTranslate="artemisApp.result.buildLogs.viewLogs"></a>
+                                    }
                                 }
                             </div>
                         }