From 73b86aa7157d5d7bd1597070b8e4af6d2dcd56c3 Mon Sep 17 00:00:00 2001 From: Angelica Ochoa <15623749+ao508@users.noreply.github.com> Date: Fri, 19 Jul 2024 20:29:36 -0400 Subject: [PATCH] Additional logging for datadaog Signed-off-by: Angelica Ochoa <15623749+ao508@users.noreply.github.com> --- .../service/CmoLabelGeneratorService.java | 2 + .../impl/CmoLabelGeneratorServiceImpl.java | 103 +++++++++++++++++- .../LabelGenMessageHandlingServiceImpl.java | 23 +++- 3 files changed, 121 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/mskcc/smile/service/CmoLabelGeneratorService.java b/src/main/java/org/mskcc/smile/service/CmoLabelGeneratorService.java index a44362f..be539a5 100644 --- a/src/main/java/org/mskcc/smile/service/CmoLabelGeneratorService.java +++ b/src/main/java/org/mskcc/smile/service/CmoLabelGeneratorService.java @@ -21,4 +21,6 @@ Status generateSampleStatus(SampleMetadata sampleMetadata, Boolean igoSampleRequiresLabelUpdate(String newCmoLabel, String existingCmoLabel); String resolveSampleTypeAbbreviation(String specimenTypeValue, String sampleOriginValue, String cmoSampleClassValue); + String generateValidationReport(String originalJson, String filteredJson) + throws JsonProcessingException; } diff --git a/src/main/java/org/mskcc/smile/service/impl/CmoLabelGeneratorServiceImpl.java b/src/main/java/org/mskcc/smile/service/impl/CmoLabelGeneratorServiceImpl.java index e23ed40..4651f8a 100644 --- a/src/main/java/org/mskcc/smile/service/impl/CmoLabelGeneratorServiceImpl.java +++ b/src/main/java/org/mskcc/smile/service/impl/CmoLabelGeneratorServiceImpl.java @@ -8,9 +8,11 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.logging.log4j.util.Strings; import org.mskcc.smile.commons.enums.CmoSampleClass; import org.mskcc.smile.commons.enums.NucleicAcid; import org.mskcc.smile.commons.enums.SampleOrigin; @@ -322,7 +324,7 @@ private String resolveNucleicAcidAbbreviation(String sampleTypeString, return "d"; } } catch (Exception e) { - LOG.debug("Could not resolve sample type acid from 'sampleType' - using default 'd'"); + LOG.warn("Could not resolve sample type acid from 'sampleType' - using default 'd'"); } // if nucleic acid abbreviation is still unknown then attempt to resolve from // sample metadata --> cmo sample id fields --> naToExtract @@ -341,7 +343,7 @@ private String resolveNucleicAcidAbbreviation(String sampleTypeString, } } } catch (Exception e) { - LOG.debug("Could not resolve nucleic acid from 'naToExtract' - using default 'd'"); + LOG.warn("Could not resolve nucleic acid from 'naToExtract' - using default 'd'"); return "d"; } @@ -383,7 +385,7 @@ public String resolveSampleTypeAbbreviation(String specimenTypeValue, String sam } } } catch (Exception e) { - LOG.debug("Could not resolve specimen type acid from 'specimenType': " + LOG.warn("Could not resolve specimen type acid from 'specimenType': " + specimenTypeValue); } @@ -564,4 +566,99 @@ private Boolean isCmoCelllineSample(IgoSampleManifest sample) { return isCmoCelllineSample(sample.getSpecimenType(), sample.getCmoSampleIdFields()); } + @Override + public String generateValidationReport(String originalJson, String filteredJson) + throws JsonProcessingException { + StringBuilder builder = new StringBuilder(); + String requestId = getRequestId(originalJson); + Map filteredJsonMap = mapper.readValue(filteredJson, Map.class); + // keeps track if there's anything to report or not. if still true after all checks + // then return null + Boolean allValid = Boolean.TRUE; + + // if request-level status is missing from the filtered json then + // a critical error likely occurred, in which case the original json + // would be more helpful to have as a reference when debugging the error + if (!filteredJsonMap.containsKey("status")) { + allValid = Boolean.FALSE; + builder.append("Request JSON missing validation report ('status') post-validation:"); + builder.append("\nOriginal JSON contents:\n") + .append(originalJson).append("\nFiltered JSON contents:\n") + .append(filteredJson); + } else { + Map statusMap = (Map) filteredJsonMap.get("status"); + Map validationReport = + mapper.convertValue(statusMap.get("validationReport"), Map.class); + + // if request validation report is not empty then log for ddog + if (!validationReport.isEmpty()) { + allValid = Boolean.FALSE; + builder.append("Request-level status and validation report for request '") + .append(requestId) + .append("': ") + .append(mapper.writeValueAsString(statusMap)); + } + // check validation status for each sample individually as well and + // add contents to report for ddog + Object[] sampleList = mapper.convertValue(filteredJsonMap.get("samples"), + Object[].class); + for (Object s : sampleList) { + Map sampleMap = mapper.convertValue(s, Map.class); + Map sampleStatusMap = mapper.convertValue(sampleMap.get("status"), Map.class); + Map sampleValidationReport = + mapper.convertValue(sampleStatusMap.get("validationReport"), Map.class); + try { + String sampleId = ObjectUtils.firstNonNull( + sampleMap.get("igoId"), sampleMap.get("primaryId")).toString(); + if (!sampleValidationReport.isEmpty()) { + allValid = Boolean.FALSE; + builder.append("\nValidation report for sample '") + .append(sampleId) + .append("': ") + .append(mapper.writeValueAsString(sampleStatusMap)); + } + } catch (NullPointerException e) { + builder.append("\nNo known identifiers in current sample data: ") + .append(mapper.writeValueAsString(sampleMap)) + .append(", Validation report for unknown sample: ") + .append(mapper.writeValueAsString(sampleStatusMap)); + } + } + } + // if allValid is still true then there wasn't anything to report at the request + // or sample level.. return null + return allValid ? null : builder.toString(); + } + + private String getRequestId(String json) throws JsonProcessingException { + if (isBlank(json)) { + return null; + } + Map jsonMap = mapper.readValue(json, Map.class); + return getRequestId(jsonMap); + } + + private String getRequestId(Map jsonMap) throws JsonProcessingException { + if (jsonMap.containsKey("requestId")) { + return jsonMap.get("requestId").toString(); + } + if (jsonMap.containsKey("igoRequestId")) { + return jsonMap.get("igoRequestId").toString(); + } + if (jsonMap.containsKey("additionalProperties")) { + Map additionalProperties = mapper.convertValue( + jsonMap.get("additionalProperties"), Map.class); + if (additionalProperties.containsKey("requestId")) { + return additionalProperties.get("requestId"); + } + if (additionalProperties.containsKey("igoRequestId")) { + return additionalProperties.get("igoRequestId"); + } + } + return null; + } + + private Boolean isBlank(String value) { + return (Strings.isBlank(value) || value.equals("null")); + } } diff --git a/src/main/java/org/mskcc/smile/service/impl/LabelGenMessageHandlingServiceImpl.java b/src/main/java/org/mskcc/smile/service/impl/LabelGenMessageHandlingServiceImpl.java index da62ba3..aaab398 100644 --- a/src/main/java/org/mskcc/smile/service/impl/LabelGenMessageHandlingServiceImpl.java +++ b/src/main/java/org/mskcc/smile/service/impl/LabelGenMessageHandlingServiceImpl.java @@ -193,8 +193,8 @@ public void run() { try { String requestJson = labelGeneratorQueue.poll(100, TimeUnit.MILLISECONDS); if (requestJson != null) { - LOG.info("Extracting samples from request received..."); String requestId = getRequestIdFromRequestJson(requestJson); + LOG.info("Extracting samples from request received: " + requestId); List samples = getSamplesFromRequestJson(requestJson); // get existing samples for all patients in the request @@ -259,12 +259,19 @@ public void run() { // and add updated request json to publisher queue Map requestJsonMap = mapper.readValue(requestJson, Map.class); requestJsonMap.put("samples", updatedSamples); + String updatedRequestJson = mapper.writeValueAsString(requestJsonMap); + // data dog log message + String ddogLogMessage = cmoLabelGeneratorService.generateValidationReport( + requestJson, updatedRequestJson); + if (ddogLogMessage != null) { + LOG.info(ddogLogMessage); + } switch (igoRequestDest) { case NEW_REQUEST_DEST: - igoNewRequestQueue.add(mapper.writeValueAsString(requestJsonMap)); + igoNewRequestQueue.add(updatedRequestJson); break; case PROMOTED_REQUEST_DEST: - igoPromotedRequestQueue.add(mapper.writeValueAsString(requestJsonMap)); + igoPromotedRequestQueue.add(updatedRequestJson); break; default: break; @@ -308,6 +315,7 @@ public void run() { try { SampleMetadata sample = cmoSampleLabelUpdateQueue.poll(100, TimeUnit.MILLISECONDS); if (sample != null) { + String origSampleJson = mapper.writeValueAsString(sample); List existingSamples = getExistingPatientSamples(sample.getCmoPatientId()); // Case when sample update json doesn't have status @@ -335,8 +343,15 @@ public void run() { sample.getPrimaryId(), existingSamples, newCmoSampleLabel); sample.setCmoSampleName(resolvedCmoSampleLabel); } + String updatedSampleJson = mapper.writeValueAsString(sample); + // data dog log message + String ddogLogMessage = cmoLabelGeneratorService.generateValidationReport( + origSampleJson, updatedSampleJson); + if (ddogLogMessage != null) { + LOG.info(ddogLogMessage); + } LOG.info("Publishing sample to IGO_SAMPLE_UPDATE_TOPIC"); - messagingGateway.publish(IGO_SAMPLE_UPDATE_TOPIC, mapper.writeValueAsString(sample)); + messagingGateway.publish(IGO_SAMPLE_UPDATE_TOPIC, updatedSampleJson); } if (interrupted && cmoSampleLabelUpdateQueue.isEmpty()) { break;