diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index abc4de5302..2a270348fb 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,6 +4,7 @@ on: push: branches: - master + workflow_dispatch: env: NODE_VERSION: "20" diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java index 765fa4d2be..ef1f49c3d1 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java @@ -76,6 +76,21 @@ public CreateJobResponse createJob(@RequestBody CreateJobRequest body) { } } + public class ExecutionResult { + private String status; + private String error; + + // Constructor + public ExecutionResult(String status, String error) { + this.status = status; + this.error = error; + } + + public ExecutionResult(String status) { + this(status, ""); + } + } + /** * Runs the validator on the GTFS file associated with the job id. The GTFS file is downloaded * from GCS, extracted locally, validated, and the results are uploaded to GCS. @@ -85,6 +100,7 @@ public ResponseEntity runValidator( @RequestBody GoogleCloudPubsubMessage googleCloudPubsubMessage) { File tempFile = null; Path outputPath = null; + String jobId = null; try { var message = googleCloudPubsubMessage.getMessage(); if (message == null) { @@ -93,24 +109,35 @@ public ResponseEntity runValidator( } ValidationJobMetaData jobData = getFeedFileMetaData(message); - var jobId = jobData.getJobId(); + jobId = jobData.getJobId(); + var fileName = jobData.getFileName(); var countryCode = storageHelper.getJobMetadata(jobId).getCountryCode(); // copy the file from GCS to a temp directory tempFile = storageHelper.downloadFeedFileFromStorage(jobId, fileName); - outputPath = storageHelper.createOutputFolderForJob(jobId); - // extracts feed files from zip to temp output directory, validates, and returns - // the path to the output directory - validationHandler.validateFeed(tempFile, outputPath, countryCode); + outputPath = storageHelper.createOutputFolderForJob(jobId); + try { + // extracts feed files from zip to temp output directory, validates + validationHandler.validateFeed(tempFile, outputPath, countryCode); + storageHelper.writeExecutionResultFile(new ExecutionResult("success"), outputPath); + } catch (Exception exc) { + logger.error("Error", exc); + Sentry.captureException(exc); + storageHelper.writeExecutionResultFile( + new ExecutionResult("error", exc.getMessage()), outputPath); + } // upload the extracted files and the validation results from outputPath to GCS storageHelper.uploadFilesToStorage(jobId, outputPath); return new ResponseEntity(HttpStatus.OK); } catch (Exception exc) { + // We are here because there was an exception in code not within the validator, i.e. probably + // related to + // cloud storage. We return 500 in that case so the GCP retry mechanism can do its magic. logger.error("Error", exc); Sentry.captureException(exc); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error", exc); diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index 4d69a8aee2..6ea6b8210a 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -3,9 +3,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.cloud.WriteChannel; import com.google.cloud.storage.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import java.io.*; import java.net.URL; import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -14,6 +17,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import org.mobilitydata.gtfsvalidator.util.HttpGetUtil; +import org.mobilitydata.gtfsvalidator.web.service.controller.ValidationController; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -36,7 +40,8 @@ public class StorageHelper { System.getenv().getOrDefault("RESULTS_BUCKET_NAME", "gtfs-validator-results"); static final String FILE_NAME = "gtfs-job.zip"; - private final Logger logger = LoggerFactory.getLogger(StorageHelper.class); + private static final Logger logger = LoggerFactory.getLogger(StorageHelper.class); + private Storage storage; private ApplicationContext applicationContext; @@ -186,4 +191,26 @@ public void uploadFilesToStorage(String jobId, Path outputPath) throws IOExcepti public Path createOutputFolderForJob(String jobId) throws IOException { return Files.createTempDirectory(StorageHelper.TEMP_FOLDER_NAME + jobId); } + + private static final String executionResultFile = "execution_result.json"; + + public void writeExecutionResultFile( + ValidationController.ExecutionResult executionResult, Path outputPath) { + if (outputPath == null) { + logger.error("Error: outputPath is null, cannot write execution result file"); + return; + } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + Path executionResultPath = outputPath.resolve(executionResultFile); + try { + logger.info("Writing executionResult file to " + executionResultFile); + Files.write( + executionResultPath, gson.toJson(executionResult).getBytes(StandardCharsets.UTF_8)); + logger.info(executionResultFile + " file written successfully"); + } catch (IOException e) { + logger.error("Error writing to file " + executionResultFile); + e.printStackTrace(); + } + } } diff --git a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/RunValidatorEndpointTest.java b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/RunValidatorEndpointTest.java index 0a269ce28f..0873a34cfb 100644 --- a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/RunValidatorEndpointTest.java +++ b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/RunValidatorEndpointTest.java @@ -1,13 +1,16 @@ package org.mobilitydata.gtfsvalidator.web.service.controller; -import static org.mockito.ArgumentMatchers.anyString; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import org.apache.commons.codec.binary.Base64; +import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mobilitydata.gtfsvalidator.web.service.util.JobMetadata; @@ -53,6 +56,25 @@ public void setUp() throws Exception { doReturn(jobMetaData).when(storageHelper).getJobMetadata(testJobId); doReturn(mockOutputPath).when(storageHelper).createOutputFolderForJob(testJobId); doReturn(mockOutputPathToFile).when(mockOutputPath).toFile(); + doReturn(Path.of("./execution_result.json")).when(mockOutputPath).resolve(anyString()); + doCallRealMethod() + .when(storageHelper) + .writeExecutionResultFile(any(ValidationController.ExecutionResult.class), any(Path.class)); + } + + public boolean executionResultIs(String result) throws Exception { + String executionResultJson = Files.readString(Paths.get("execution_result.json")); + JSONObject executionResult = new JSONObject(executionResultJson); + String expectedStatus = executionResult.getString("status"); + return result.equals(expectedStatus); + } + + public boolean executionResultIsError() throws Exception { + return executionResultIs("error"); + } + + public boolean executionResultIsSuccess() throws Exception { + return executionResultIs("success"); } @Test @@ -68,6 +90,8 @@ public void runValidatorSuccess() throws Exception { .contentType(MediaType.APPLICATION_JSON)) .andExpect(MockMvcResultMatchers.status().isOk()); + assertTrue(executionResultIsSuccess()); + // verify that the validationHandler is called with the downloaded feed file, output path, and // country code verify(validationHandler, times(1)) @@ -94,13 +118,13 @@ public void runValidatorStorageDownloadFailure() throws Exception { .contentType(MediaType.APPLICATION_JSON)) .andExpect(MockMvcResultMatchers.status().is5xxServerError()); - // should not attempt validation + // should not have attempted validation verify(validationHandler, times(0)).validateFeed(any(File.class), any(Path.class), anyString()); - // should not upload to storage + // should not have uploaded to storage verify(storageHelper, times(0)).uploadFilesToStorage(anyString(), any(Path.class)); - // should not delete temp files + // should not have tried to delete temp files verify(mockFeedFile, times(0)).delete(); verify(mockOutputPathToFile, times(0)).delete(); } @@ -120,13 +144,8 @@ public void runValidatorValidateFeedFailure() throws Exception { MockMvcRequestBuilders.post("/run-validator") .content(mapper.writeValueAsString(pubSubMessage)) .contentType(MediaType.APPLICATION_JSON)) - .andExpect(MockMvcResultMatchers.status().is5xxServerError()); - - // should not upload to storage - verify(storageHelper, times(0)).uploadFilesToStorage(anyString(), any(Path.class)); + .andExpect(MockMvcResultMatchers.status().isOk()); - // should delete temp files when an exception is thrown - verify(mockFeedFile, times(1)).delete(); - verify(mockOutputPathToFile, times(1)).delete(); + assertTrue(executionResultIsError()); } }