From 3360718ab90612c707250b7439ba67db17d82082 Mon Sep 17 00:00:00 2001 From: gnefedev Date: Wed, 14 Feb 2024 14:58:50 +0100 Subject: [PATCH 1/6] CommandReduceToCommonAssembleFeature, first version --- .../milaboratory/mixcr/cli/CommandAlign.kt | 12 +- .../milaboratory/mixcr/cli/CommandAnalyze.kt | 31 +- .../milaboratory/mixcr/cli/CommandAssemble.kt | 12 +- .../mixcr/cli/CommandAssembleContigs.kt | 16 +- .../mixcr/cli/CommandAssemblePartial.kt | 10 +- .../mixcr/cli/CommandExportAlignments.kt | 6 +- .../mixcr/cli/CommandExportCloneGroups.kt | 6 +- .../mixcr/cli/CommandExportClones.kt | 6 +- .../mixcr/cli/CommandExportPreset.kt | 26 +- .../mixcr/cli/CommandExportReports.kt | 6 +- .../mixcr/cli/CommandExportReportsAsTable.kt | 8 +- .../mixcr/cli/CommandExportSchemas.kt | 6 +- .../milaboratory/mixcr/cli/CommandExtend.kt | 14 +- .../mixcr/cli/CommandFindAlleles.kt | 39 +- .../mixcr/cli/CommandFindShmTrees.kt | 3 +- .../mixcr/cli/CommandGroupClones.kt | 10 +- .../milaboratory/mixcr/cli/CommandQcChecks.kt | 6 +- .../CommandReduceToCommonAssembleFeature.kt | 440 ++++++++++++++++++ .../mixcr/cli/CommandRefineTagsAndSort.kt | 10 +- .../kotlin/com/milaboratory/mixcr/cli/Main.kt | 4 + .../mixcr/cli/MiXCRParamsResolver.kt | 10 +- .../com/milaboratory/mixcr/cli/Mixins.kt | 8 +- .../milaboratory/mixcr/cli/OutputTemplate.kt | 42 ++ .../mixcr/cli/qc/CommandExportQcTags.kt | 6 +- .../com/milaboratory/mixcr/PresetsTest.kt | 10 +- .../mixcr/cli/CommandExportPresetTest.kt | 12 +- .../milaboratory/mixcr/cli/MetaInfoTest.kt | 15 +- 27 files changed, 621 insertions(+), 153 deletions(-) create mode 100644 src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt create mode 100644 src/main/kotlin/com/milaboratory/mixcr/cli/OutputTemplate.kt diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAlign.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAlign.kt index 5478babe8..508c8e637 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAlign.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAlign.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -73,9 +73,9 @@ import com.milaboratory.mixcr.cli.CommonDescriptions.Labels import com.milaboratory.mixcr.cli.MiXCRCommand.OptionsOrder import com.milaboratory.mixcr.presets.AlignMixins import com.milaboratory.mixcr.presets.AlignMixins.LimitInput +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor.Companion.dotAfterIfNotBlank import com.milaboratory.mixcr.presets.FullSampleSheetParsed -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor.Companion.dotAfterIfNotBlank import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRParamsSpec import com.milaboratory.mixcr.presets.MiXCRStepParams @@ -127,7 +127,7 @@ import kotlin.io.path.readText import kotlin.math.max object CommandAlign { - const val COMMAND_NAME = MiXCRCommandDescriptor.align.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.align.name const val SAVE_OUTPUT_FILE_NAMES_OPTION = "--save-output-file-names" const val STRICT_SAMPLE_NAME_MATCHING_OPTION = "--strict-sample-sheet-matching" @@ -1165,7 +1165,7 @@ object CommandAlign { MiXCRHeader( inputHash, dontSavePresetOption.presetToSave(paramsSpecPacked), - MiXCRStepParams().add(MiXCRCommandDescriptor.align, cmdParams), + MiXCRStepParams().add(AnalyzeCommandDescriptor.align, cmdParams), tagsExtractor.tagsInfo, aligner.parameters, aligner.parameters.featuresToAlignMap, @@ -1306,7 +1306,7 @@ object CommandAlign { reportBuilder.setTransformerReports(tagsExtractor.transformerReports) val report = reportBuilder.buildReport() - writers?.setFooter(MiXCRFooter().addStepReport(MiXCRCommandDescriptor.align, report)) + writers?.setFooter(MiXCRFooter().addStepReport(AnalyzeCommandDescriptor.align, report)) // Writing report to stout ReportUtil.writeReportToStdout(report) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAnalyze.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAnalyze.kt index 0cb347199..1eea05866 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAnalyze.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAnalyze.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -22,10 +22,9 @@ import com.milaboratory.mixcr.cli.CommandAlign.inputFileGroups import com.milaboratory.mixcr.cli.CommandAlign.listSamplesForSeedFileName import com.milaboratory.mixcr.cli.CommonDescriptions.Labels import com.milaboratory.mixcr.presets.AlignMixins -import com.milaboratory.mixcr.presets.AnyMiXCRCommand +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor.Companion.dotAfterIfNotBlank import com.milaboratory.mixcr.presets.FullSampleSheetParsed -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor.Companion.dotAfterIfNotBlank import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRParamsSpec import com.milaboratory.mixcr.presets.MiXCRPipeline @@ -338,7 +337,7 @@ object CommandAnalyze { .let { (first, second) -> first to second.steps.sortedBy { it.order } } // Creating execution plan - if (pipeline[0] != MiXCRCommandDescriptor.align) + if (pipeline[0] != AnalyzeCommandDescriptor.align) throw ValidationException("Pipeline must stat from the align action.") val planBuilder = PlanBuilder( @@ -365,7 +364,7 @@ object CommandAnalyze { // Adding "align" step if (outputNoUsedReads) pathsForNotAligned.fillWithDefaults(inputFileGroups.inputType, outputFolder, outputNamePrefix) - planBuilder.addStep(MiXCRCommandDescriptor.align) { _, _, _ -> + planBuilder.addStep(AnalyzeCommandDescriptor.align) { _, _, _ -> listOf("--preset", presetName) + extraAlignArgs + mixins.flatMap { it.cmdArgs } + pathsForNotAligned.argsForAlign() } @@ -381,24 +380,24 @@ object CommandAnalyze { // Adding all steps with calculations pipeline .drop(1) - .filter { cmd -> cmd !is MiXCRCommandDescriptor.ExportCommandDescriptor } + .filter { cmd -> cmd !is AnalyzeCommandDescriptor.ExportCommandDescriptor } .forEach { cmd -> planBuilder.addStep(cmd) { outputFolder, prefix, sampleName -> when (cmd) { - MiXCRCommandDescriptor.assemble -> { + AnalyzeCommandDescriptor.assemble -> { val additionalArgs = mutableListOf() if (consensusAlignments) additionalArgs += listOf( "--consensus-alignments", outputFolder.resolve( - MiXCRCommandDescriptor.assemble.consensusAlignments(prefix, sampleName) + AnalyzeCommandDescriptor.assemble.consensusAlignments(prefix, sampleName) ).toString() ) if (consensusStateStats) additionalArgs += listOf( "--consensus-state-stat", outputFolder.resolve( - MiXCRCommandDescriptor.assemble.consensusStateStats(prefix, sampleName) + AnalyzeCommandDescriptor.assemble.consensusStateStats(prefix, sampleName) ).toString() ) consensusStateStatsDownsampling?.let { @@ -417,7 +416,7 @@ object CommandAnalyze { } pipeline - .filterIsInstance>() + .filterIsInstance>() .forEach { cmd -> planBuilder.addExportStep(cmd) } @@ -442,9 +441,9 @@ object CommandAnalyze { private val forceOverride: Boolean ) { private val executionPlan = mutableListOf() - private val rounds = mutableMapOf() + private val rounds = mutableMapOf, Int>() private var nextInputs: List = listOf(InputFileSet("", initialInputs.map { it.toString() })) - private val outputsForCommands = mutableListOf>>() + private val outputsForCommands = mutableListOf, List>>() fun setActualAlignOutputs(fileNames: List) { val outputSeed = Path(nextInputs.requireSingleton().fileNames.requireSingleton()).name @@ -483,7 +482,7 @@ object CommandAnalyze { for (input in inputsForQc) { check(input.fileNames.size == 1) val round = 0 - val cmd = MiXCRCommandDescriptor.qc + val cmd = AnalyzeCommandDescriptor.qc val outputName = cmd.outputName(outputNamePrefix, input.sampleName, paramsBundle, round) val arguments = mutableListOf("--print-to-stdout") if (forceOverride) @@ -503,7 +502,7 @@ object CommandAnalyze { } } - fun addExportStep(cmd: MiXCRCommandDescriptor.ExportCommandDescriptor<*>) { + fun addExportStep(cmd: AnalyzeCommandDescriptor.ExportCommandDescriptor<*>) { val runAfter = cmd.runAfterLastOf() val inputsForExport = outputsForCommands.findLast { (cmd) -> cmd in runAfter }!!.second for (input in inputsForExport) { @@ -527,7 +526,7 @@ object CommandAnalyze { } fun addStep( - cmd: AnyMiXCRCommand, + cmd: AnalyzeCommandDescriptor<*, *>, extraArgs: (outputFolder: Path, prefix: String, sampleName: String) -> List = { _, _, _ -> emptyList() } ) { val round = rounds.compute(cmd) { c, p -> diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemble.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemble.kt index b6753f15a..cd4c9cca7 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemble.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemble.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -42,7 +42,7 @@ import com.milaboratory.mixcr.basictypes.tag.TagType import com.milaboratory.mixcr.basictypes.validateCompositeFeatures import com.milaboratory.mixcr.cli.CommonDescriptions.DEFAULT_VALUE_FROM_PRESET import com.milaboratory.mixcr.cli.CommonDescriptions.Labels -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.util.ArraysUtils import com.milaboratory.util.HashFunctions @@ -66,7 +66,7 @@ import kotlin.io.path.bufferedWriter import kotlin.io.path.extension object CommandAssemble { - const val COMMAND_NAME = MiXCRCommandDescriptor.assemble.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.assemble.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -388,7 +388,7 @@ object CommandAssemble { val resultHeader = inputHeader .withAssemblerParameters(cloneAssemblerParameters) .addStepParams( - MiXCRCommandDescriptor.assemble, + AnalyzeCommandDescriptor.assemble, paramsResolver.resolve(paramSpec).second ) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramSpec)) @@ -440,7 +440,7 @@ object CommandAssemble { report = reportBuilder.buildReport() writer.setFooter( alignmentsReader.footer.addStepReport( - MiXCRCommandDescriptor.assemble, + AnalyzeCommandDescriptor.assemble, report ) ) @@ -453,7 +453,7 @@ object CommandAssemble { report = reportBuilder.buildReport() writer.setFooter( alignmentsReader.footer.addStepReport( - MiXCRCommandDescriptor.assemble, + AnalyzeCommandDescriptor.assemble, report ) ) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt index 898925ad2..fbfc7bf43 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt @@ -36,7 +36,7 @@ import com.milaboratory.mixcr.basictypes.VDJCSProperties.CloneOrdering import com.milaboratory.mixcr.basictypes.validateCompositeFeatures import com.milaboratory.mixcr.cli.CommonDescriptions.DEFAULT_VALUE_FROM_PRESET import com.milaboratory.mixcr.cli.CommonDescriptions.Labels -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRParamsSpec import com.milaboratory.mixcr.util.Concurrency @@ -66,7 +66,7 @@ import java.nio.file.Path import java.util.* object CommandAssembleContigs { - const val COMMAND_NAME = MiXCRCommandDescriptor.assembleContigs.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.assembleContigs.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -172,11 +172,11 @@ object CommandAssembleContigs { validateParams(cmdParams, reader.header) - require(reader.assemblingFeatures.size == 1) { + ValidationException.require(reader.assemblingFeatures.size == 1) { "Supports only singular assemblingFeature." } val assemblingFeature = reader.assemblingFeatures.first() - require(!assemblingFeature.isComposite) { + ValidationException.require(!assemblingFeature.isComposite) { "Supports only non-composite gene features as an assemblingFeature." } @@ -184,7 +184,7 @@ object CommandAssembleContigs { val fullyIncluded = assemblingRegions.features.any { assemblingRegion -> GeneFeature.intersection(assemblingRegion, assemblingFeature) == assemblingFeature } - require(fullyIncluded) { + ValidationException.require(fullyIncluded) { "AssemblingFeature of input must be included fully in assemblingRegions" } } @@ -305,10 +305,10 @@ object CommandAssembleContigs { clones += clone.withId(cloneId++) } } - val allFullyCoveredBy = cmdParams.allClonesWillBeCoveredByFeature() + val allFullyCoveredBy = cmdParams.parameters.allClonesWillBeCoveredByFeature() val resultHeader = header .copy(allFullyCoveredBy = if (allFullyCoveredBy) assemblingRegions else null) - .addStepParams(MiXCRCommandDescriptor.assembleContigs, cmdParams) + .addStepParams(AnalyzeCommandDescriptor.assembleContigs, cmdParams) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)) val cloneSet = CloneSet.Builder(clones, genes, resultHeader) @@ -327,7 +327,7 @@ object CommandAssembleContigs { // Writing report to stout ReportUtil.writeReportToStdout(report) reportOptions.appendToFiles(report) - writer.setFooter(footer.addStepReport(MiXCRCommandDescriptor.assembleContigs, report)) + writer.setFooter(footer.addStepReport(AnalyzeCommandDescriptor.assembleContigs, report)) } } } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemblePartial.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemblePartial.kt index 1ac831528..7ec8ec70f 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemblePartial.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssemblePartial.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -25,7 +25,7 @@ import com.milaboratory.mixcr.basictypes.tag.TagType import com.milaboratory.mixcr.cli.CommonDescriptions.DEFAULT_VALUE_FROM_PRESET import com.milaboratory.mixcr.cli.CommonDescriptions.Labels import com.milaboratory.mixcr.partialassembler.PartialAlignmentsAssembler -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.util.ReportUtil import com.milaboratory.util.SmartProgressReporter @@ -38,7 +38,7 @@ import picocli.CommandLine.Parameters import java.nio.file.Path object CommandAssemblePartial { - const val COMMAND_NAME = MiXCRCommandDescriptor.assemblePartial.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.assemblePartial.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -144,7 +144,7 @@ object CommandAssemblePartial { writer.writeHeader( header .updateTagInfo { ti -> ti.setSorted(groupingDepth) } // output data will be grouped only up to a groupingDepth - .addStepParams(MiXCRCommandDescriptor.assemblePartial, cmdParams) + .addStepParams(AnalyzeCommandDescriptor.assemblePartial, cmdParams) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)), reader.usedGenes ) @@ -203,7 +203,7 @@ object CommandAssemblePartial { } reportOptions.appendToFiles(report) writer.setNumberOfProcessedReads(reader.numberOfReads - assembler.overlapped.get()) - writer.setFooter(reader.footer.addStepReport(MiXCRCommandDescriptor.assemblePartial, report)) + writer.setFooter(reader.footer.addStepReport(AnalyzeCommandDescriptor.assemblePartial, report)) } } } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportAlignments.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportAlignments.kt index aacccf7ba..eba41db47 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportAlignments.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportAlignments.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -30,7 +30,7 @@ import com.milaboratory.mixcr.export.InfoWriter import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.RowMetaForExport import com.milaboratory.mixcr.export.VDJCAlignmentsFieldsExtractorsFactory -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.util.Concurrency import com.milaboratory.util.OutputPortWithProgress @@ -48,7 +48,7 @@ import picocli.CommandLine.Parameters import java.nio.file.Path object CommandExportAlignments { - const val COMMAND_NAME = MiXCRCommandDescriptor.exportAlignments.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.exportAlignments.name fun CommandExportAlignmentsParams.mkFilter(): Filter { val chainsParsed = Chains.parse(chains) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportCloneGroups.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportCloneGroups.kt index 9a32cb92a..b8b9e647d 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportCloneGroups.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportCloneGroups.kt @@ -32,8 +32,8 @@ import com.milaboratory.mixcr.export.CloneGroupFieldsExtractorsFactory import com.milaboratory.mixcr.export.InfoWriter import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.RowMetaForExport +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.ExportMixins -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.util.SubstitutionHelper import com.milaboratory.util.asOutputPortWithProgress @@ -49,7 +49,7 @@ import java.nio.file.Path import java.nio.file.Paths object CommandExportCloneGroups { - const val COMMAND_NAME = MiXCRCommandDescriptor.exportCloneGroups.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.exportCloneGroups.name private fun CommandExportCloneGroupsParams.test( clone: Clone, @@ -116,7 +116,7 @@ object CommandExportCloneGroups { } @Command( - description = ["Export clone groups into tab delimited file. Data should be processed by `${MiXCRCommandDescriptor.assembleCells.name}`"] + description = ["Export clone groups into tab delimited file. Data should be processed by `${AnalyzeCommandDescriptor.assembleCells.name}`"] ) class Cmd : CmdBase() { @Parameters( diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportClones.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportClones.kt index 2ba2b51ea..718055802 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportClones.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportClones.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -36,7 +36,7 @@ import com.milaboratory.mixcr.export.CloneFieldsExtractorsFactory import com.milaboratory.mixcr.export.InfoWriter import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.RowMetaForExport -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.util.SubstitutionHelper import com.milaboratory.util.ReportHelper @@ -55,7 +55,7 @@ import java.nio.file.Path import kotlin.io.path.Path object CommandExportClones { - const val COMMAND_NAME = MiXCRCommandDescriptor.exportClones.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.exportClones.name private fun CommandExportClonesParams.test( clone: Clone, diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportPreset.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportPreset.kt index 716ac150d..7409470cb 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportPreset.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportPreset.kt @@ -27,7 +27,7 @@ import com.milaboratory.mixcr.basictypes.IOUtil.MiXCRFileType.SHMT import com.milaboratory.mixcr.basictypes.IOUtil.MiXCRFileType.VDJCA import com.milaboratory.mixcr.basictypes.VDJCAlignmentsReader import com.milaboratory.mixcr.cli.CommonDescriptions.Labels -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParams import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRParamsSpec @@ -184,7 +184,7 @@ class CommandExportPreset : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand> paramsWithOverride( + fun

> paramsWithOverride( descriptor: T ): P? = header.stepParams[descriptor].firstOrNull() ?: descriptor.extractFromBundle(originalPreset) @@ -192,17 +192,17 @@ class CommandExportPreset : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand footer.reports.collection.steps } steps.forEach { step -> - val command = MiXCRCommandDescriptor.fromString(step) + val command = AnalyzeCommandDescriptor.fromString(step) val reports = footer.reports.getReportSafe(command) if (reports == null) { helper.println("Can't read report for $step, file has too old version") diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt index 118635ae7..01a23eef9 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -23,7 +23,7 @@ import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.ReportFieldsExtractors import com.milaboratory.mixcr.export.ReportFieldsExtractors.ReportsWithSource import com.milaboratory.mixcr.export.RowMetaForExport -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.StepDataCollection import com.milaboratory.mixcr.presets.getReportSafe import picocli.CommandLine @@ -121,7 +121,7 @@ class CommandExportReportsAsTable : MiXCRCommandWithOutputs() { } upstreamRecords + ReportsWithSource( input.toString(), - MiXCRCommandDescriptor.values().mapNotNull { footer.reports.getReportSafe(it) }.flatten(), + AnalyzeCommandDescriptor.values().mapNotNull { footer.reports.getReportSafe(it) }.flatten(), upstreamRecords.flatMap { it.reports } ) } @@ -139,7 +139,7 @@ class CommandExportReportsAsTable : MiXCRCommandWithOutputs() { val upstreamReports = collection.upstreamReportsWithSources() upstreamReports + ReportsWithSource( sourceName, - MiXCRCommandDescriptor.values().mapNotNull { collection.getReportSafe(it) }.flatten(), + AnalyzeCommandDescriptor.values().mapNotNull { collection.getReportSafe(it) }.flatten(), upstreamReports.flatMap { it.reports } ) } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportSchemas.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportSchemas.kt index 41c171f5c..fc8637226 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportSchemas.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportSchemas.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -25,7 +25,7 @@ import com.milaboratory.mixcr.assembler.preclone.PreCloneAssemblerParameters import com.milaboratory.mixcr.partialassembler.PartialAlignmentsAssemblerParameters import com.milaboratory.mixcr.postanalysis.ui.PostanalysisParametersIndividual import com.milaboratory.mixcr.postanalysis.ui.PostanalysisParametersOverlap -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRMixin import com.milaboratory.mixcr.trees.CommandFindShmTreesParams import com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters @@ -112,7 +112,7 @@ class CommandExportSchemas : Runnable { val analyzeBundleDir = outputPath.resolve("analyzeBundle") val parametersDir = analyzeBundleDir.resolve("parameters") parametersDir.toFile().mkdirs() - for (command in MiXCRCommandDescriptor::class.sealedFinalSubclasses()) { + for (command in AnalyzeCommandDescriptor::class.sealedFinalSubclasses()) { val parameters = command.objectInstance?.paramClass!! K_YAML_OM.writeValue( parametersDir.resolve("${command.simpleName}.schema.yaml").toFile(), diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExtend.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExtend.kt index 427558620..b7bcf9c38 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExtend.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExtend.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -33,7 +33,7 @@ import com.milaboratory.mixcr.basictypes.VDJCAlignmentsWriter import com.milaboratory.mixcr.basictypes.VDJCObject import com.milaboratory.mixcr.cli.CommonDescriptions.DEFAULT_VALUE_FROM_PRESET import com.milaboratory.mixcr.cli.CommonDescriptions.Labels -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRParamsSpec import com.milaboratory.mixcr.util.VDJCObjectExtender @@ -53,7 +53,7 @@ import java.nio.file.Path object CommandExtend { - const val COMMAND_NAME = MiXCRCommandDescriptor.extend.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.extend.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -185,7 +185,7 @@ object CommandExtend { process.output.toList(), cloneSet.usedGenes, cloneSet.header - .addStepParams(MiXCRCommandDescriptor.extend, process.params) + .addStepParams(AnalyzeCommandDescriptor.extend, process.params) .copy(allFullyCoveredBy = null) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)) ) @@ -195,7 +195,7 @@ object CommandExtend { ClnsWriter(outputFile).use { writer -> writer.writeCloneSet(newCloneSet) val report = process.finish() - writer.setFooter(reader.footer.addStepReport(MiXCRCommandDescriptor.extend, report)) + writer.setFooter(reader.footer.addStepReport(AnalyzeCommandDescriptor.extend, report)) } } } @@ -209,7 +209,7 @@ object CommandExtend { writer.writeHeader( reader.header .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)) - .addStepParams(MiXCRCommandDescriptor.extend, process.params), + .addStepParams(AnalyzeCommandDescriptor.extend, process.params), reader.usedGenes ) writer.setFooter(reader.footer) @@ -226,7 +226,7 @@ object CommandExtend { writer.setNumberOfProcessedReads(reader.numberOfReads) val report = process.finish() - writer.setFooter(reader.footer.addStepReport(MiXCRCommandDescriptor.extend, report)) + writer.setFooter(reader.footer.addStepReport(AnalyzeCommandDescriptor.extend, report)) } } } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindAlleles.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindAlleles.kt index 4b161ccaf..f9019b3dc 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindAlleles.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindAlleles.kt @@ -75,12 +75,10 @@ import picocli.CommandLine.Option import picocli.CommandLine.Parameters import java.io.File import java.nio.file.Path -import java.nio.file.Paths import kotlin.collections.set import kotlin.io.path.createDirectories import kotlin.io.path.isDirectory import kotlin.io.path.listDirectoryEntries -import kotlin.io.path.nameWithoutExtension @Command( description = [ @@ -110,12 +108,7 @@ class CommandFindAlleles : MiXCRCommandWithOutputs() { class OutputClnsOptions { @Option( - description = [ - "Output template may contain {file_name} and {file_dir_path},", - "outputs for '-o /output/folder/{file_name}_with_alleles.clns input_file.clns input_file2.clns' will be /output/folder/input_file_with_alleles.clns and /output/folder/input_file2_with_alleles.clns,", - "outputs for '-o {file_dir_path}/{file_name}_with_alleles.clns /some/folder1/input_file.clns /some/folder2/input_file2.clns' will be /seme/folder1/input_file_with_alleles.clns and /some/folder2/input_file2_with_alleles.clns", - "Resulted outputs must be uniq" - ], + description = [OutputTemplate.description], names = ["--output-template"], paramLabel = "", required = true, @@ -200,7 +193,7 @@ class CommandFindAlleles : MiXCRCommandWithOutputs() { @Option( names = ["-O"], - description = ["Overrides default build SHM parameter values"], + description = ["Overrides default find alleles parameter values"], paramLabel = Labels.OVERRIDES, order = OptionsOrder.overrides ) @@ -215,24 +208,7 @@ class CommandFindAlleles : MiXCRCommandWithOutputs() { private val outputClnsFiles: List by lazy { val template = outputClnsOptions.outputTemplate ?: return@lazy emptyList() - if (!template.endsWith(".clns")) { - throw ValidationException("Wrong template: command produces only clns, got $template") - } - val clnsFiles = inputFiles - .map { it.toAbsolutePath() } - .map { path -> - template - .replace(Regex("\\{file_name}"), path.nameWithoutExtension) - .replace(Regex("\\{file_dir_path}"), path.parent.toString()) - } - .map { Paths.get(it) } - .toList() - if (clnsFiles.distinct().count() < clnsFiles.size) { - var message = "Output clns files are not uniq: $clnsFiles" - message += "\nTry to use `{file_name}` and/or `{file_dir_path}` in template to get different output paths for every input. See help for more details" - throw ValidationException(message) - } - clnsFiles + OutputTemplate.calculateOutputs(template, inputFiles) } public override val outputFiles get() = outputClnsFiles + listOfNotNull(allelesMutationsOutput) + libraryOutputs @@ -262,6 +238,12 @@ class CommandFindAlleles : MiXCRCommandWithOutputs() { libraryOutputs.forEach { output -> ValidationException.requireFileType(output, InputFileType.JSON, InputFileType.FASTA) } + outputClnsOptions.outputTemplate?.let { template -> + if (!template.endsWith(".clns")) { + throw ValidationException("Wrong template: command produces only clns, got $template") + } + } + if ((listOfNotNull(outputClnsOptions.outputTemplate, allelesMutationsOutput) + libraryOutputs).isEmpty()) { throw ValidationException("--output-template, --export-library or --export-alleles-mutations must be set") } @@ -345,8 +327,7 @@ class CommandFindAlleles : MiXCRCommandWithOutputs() { val baseGenes = originalLibrary.allGenes .mapNotNull { it.data.alleleInfo?.parent } .distinct() - val absentBaseGenes = baseGenes.filter { it !in originalLibrary } - ValidationException.requireEmpty(absentBaseGenes) { + ValidationException.requireEmpty(baseGenes.filter { it !in originalLibrary }) { "All base genes must be presented in the library" } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindShmTrees.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindShmTrees.kt index 512544ed5..1044fdd05 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindShmTrees.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandFindShmTrees.kt @@ -33,6 +33,7 @@ import com.milaboratory.mixcr.basictypes.tag.TagType import com.milaboratory.mixcr.basictypes.tag.TagsInfo import com.milaboratory.mixcr.basictypes.validateCompositeFeatures import com.milaboratory.mixcr.cli.CommonDescriptions.Labels +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.AssembleContigsMixins import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsSpec @@ -411,7 +412,7 @@ class CommandFindShmTrees : MiXCRCommandWithOutputs() { val filesToGroup = inputFiles .filterIndexed { index, _ -> index in datasetsThatShouldHaveGroups } .joinToString(", ") - "Can't use information about cell, run `${MiXCRCommandDescriptor.assembleCells}` for $filesToGroup" + "Can't use information about cell, run `${AnalyzeCommandDescriptor.assembleCells}` for $filesToGroup" } } return datasetsWithGroups diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandGroupClones.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandGroupClones.kt index 8c8bff905..202df6867 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandGroupClones.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandGroupClones.kt @@ -30,7 +30,7 @@ import com.milaboratory.mixcr.basictypes.MiXCRHeader import com.milaboratory.mixcr.basictypes.VDJCAlignments import com.milaboratory.mixcr.basictypes.tag.TagType import com.milaboratory.mixcr.clonegrouping.CloneGroupingParams.Companion.mkGrouper -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.util.Concurrency import com.milaboratory.util.ReportUtil @@ -44,7 +44,7 @@ import picocli.CommandLine.Parameters import java.nio.file.Path object CommandGroupClones { - const val COMMAND_NAME = MiXCRCommandDescriptor.assembleCells.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.assembleCells.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -129,7 +129,7 @@ object CommandGroupClones { writer.writeCloneSet(result) reportBuilder.setFinishMillis(System.currentTimeMillis()) val report = reportBuilder.buildReport() - writer.setFooter(reader.footer.addStepReport(MiXCRCommandDescriptor.assembleCells, report)) + writer.setFooter(reader.footer.addStepReport(AnalyzeCommandDescriptor.assembleCells, report)) report } } @@ -154,7 +154,7 @@ object CommandGroupClones { writer.collateAlignments(allAlignmentsList.flatten(), newNumberOfAlignments) reportBuilder.setFinishMillis(System.currentTimeMillis()) val report = reportBuilder.buildReport() - writer.setFooter(reader.footer.addStepReport(MiXCRCommandDescriptor.assembleCells, report)) + writer.setFooter(reader.footer.addStepReport(AnalyzeCommandDescriptor.assembleCells, report)) writer.writeAlignmentsAndIndex() report } @@ -195,7 +195,7 @@ object CommandGroupClones { input.usedGenes, input.header .copy(calculatedCloneGroups = true) - .addStepParams(MiXCRCommandDescriptor.assembleCells, cmdParams) + .addStepParams(AnalyzeCommandDescriptor.assembleCells, cmdParams) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)) ) .sort(input.ordering) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandQcChecks.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandQcChecks.kt index 1e8198b47..4f6ef2c18 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandQcChecks.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandQcChecks.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -19,7 +19,7 @@ import com.milaboratory.cli.POverridesBuilderOps import com.milaboratory.cli.ParamsResolver import com.milaboratory.mixcr.basictypes.IOUtil import com.milaboratory.mixcr.cli.MiXCRMixinCollection.Companion.mixins -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.qc.checks.QcCheckResult import com.milaboratory.mixcr.qc.checks.QcChecker.QualityStatus.ALERT @@ -36,7 +36,7 @@ import java.io.PrintStream import java.nio.file.Path object CommandQcChecks { - const val COMMAND_NAME = MiXCRCommandDescriptor.qc.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.qc.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Mixin diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt new file mode 100644 index 000000000..0361aa007 --- /dev/null +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved + * + * Before downloading or accessing the software, please read carefully the + * License Agreement available at: + * https://github.com/milaboratory/mixcr/blob/develop/LICENSE + * + * By downloading or accessing the software, you accept and agree to be bound + * by the terms of the License Agreement. If you do not want to agree to the terms + * of the Licensing Agreement, you must not download or access the software. + */ +package com.milaboratory.mixcr.cli + +import cc.redberry.pipe.CUtils +import cc.redberry.pipe.util.buffered +import cc.redberry.pipe.util.filter +import cc.redberry.pipe.util.map +import cc.redberry.pipe.util.mapInParallel +import cc.redberry.pipe.util.toList +import com.milaboratory.app.InputFileType +import com.milaboratory.app.ValidationException +import com.milaboratory.app.logger +import com.milaboratory.core.alignment.Alignment +import com.milaboratory.mixcr.assembler.CloneFactory +import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssembler +import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerParameters +import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerReportBuilder +import com.milaboratory.mixcr.basictypes.ClnsWriter +import com.milaboratory.mixcr.basictypes.Clone +import com.milaboratory.mixcr.basictypes.CloneSet +import com.milaboratory.mixcr.basictypes.CloneSetIO +import com.milaboratory.mixcr.basictypes.VDJCAlignments +import com.milaboratory.mixcr.basictypes.VDJCHit +import com.milaboratory.mixcr.basictypes.tag.TagCount +import com.milaboratory.mixcr.cli.CommonDescriptions.Labels +import com.milaboratory.mixcr.presets.AssembleContigsMixins +import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.trees.constructStateBuilder +import com.milaboratory.mixcr.util.VJPair +import com.milaboratory.mixcr.util.plus +import com.milaboratory.util.ComparatorWithHash +import com.milaboratory.util.FormatUtils +import com.milaboratory.util.JsonOverrider +import com.milaboratory.util.ReportUtil +import com.milaboratory.util.TempFileDest +import com.milaboratory.util.TempFileManager +import com.milaboratory.util.groupByOnDisk +import io.repseq.core.GeneFeature.CDR3 +import io.repseq.core.GeneFeature.VDJRegion +import io.repseq.core.GeneFeatures +import io.repseq.core.GeneType +import io.repseq.core.GeneType.Constant +import io.repseq.core.GeneType.Diversity +import io.repseq.core.GeneType.Joining +import io.repseq.core.GeneType.Variable +import io.repseq.core.ReferencePoint.CDR3Begin +import io.repseq.core.ReferencePoint.CDR3End +import io.repseq.core.VDJCLibraryRegistry +import picocli.CommandLine +import picocli.CommandLine.Command +import picocli.CommandLine.Help.Visibility.ALWAYS +import picocli.CommandLine.Mixin +import picocli.CommandLine.Model.CommandSpec +import picocli.CommandLine.Option +import picocli.CommandLine.Parameters +import java.nio.file.Path +import java.nio.file.Paths +import java.time.Duration +import java.time.Instant +import java.util.* +import java.util.concurrent.atomic.LongAdder +import kotlin.io.path.copyTo +import kotlin.io.path.createDirectories +import kotlin.io.path.deleteIfExists +import kotlin.io.path.isDirectory +import kotlin.io.path.listDirectoryEntries + +@Command( + description = [ + "Reassemble clones so, that all outputs will have the same assemble feature, thus could be used for inferring alleles and building SHM trees." + ] +) +class CommandReduceToCommonAssembleFeature : MiXCRCommandWithOutputs() { + @Parameters( + index = "0", + arity = "2..*", + paramLabel = "$inputsLabel $outputLabel", + hideParamSyntax = true, + // help is covered by mkCommandSpec + hidden = true + ) + val inOut: List = mutableListOf() + + @Option( + names = ["--maximum-gene-feature"], + required = false, + paramLabel = Labels.GENE_FEATURE, + description = ["Limit result intersected feature by this."], + showDefaultValue = ALWAYS + ) + var maximumGeneFeature: GeneFeatures = GeneFeatures(VDJRegion) + + @Mixin + lateinit var threadsOptions: ThreadsOption + + @Mixin + lateinit var useLocalTemp: UseLocalTempOption + + @Mixin + lateinit var reportOptions: ReportOptions + + @Option( + names = ["-O"], + description = ["Overrides default find alleles parameter values"], + paramLabel = Labels.OVERRIDES, + order = OptionsOrder.overrides + ) + var overrides: Map = mutableMapOf() + + private val outputTemplate get() = inOut.last() + + override val inputFiles: List + get() = inOut.dropLast(1).map { Paths.get(it) }.flatMap { path -> + when { + path.isDirectory() -> path.listDirectoryEntries() + else -> listOf(path) + } + } + + override val outputFiles: List by lazy { + OutputTemplate.calculateOutputs(outputTemplate, inputFiles) + } + + private val tempDest: TempFileDest by lazy { + val path = outputFiles.first() + if (useLocalTemp.value) path.toAbsolutePath().parent.createDirectories() + TempFileManager.smartTempDestination(path, ".cutToCommonFeature", !useLocalTemp.value) + } + + private fun assemblerParameters(targetFeature: GeneFeatures): FullSeqAssemblerParameters { + val presetName = "default" + var result = FullSeqAssemblerParameters.presets.getByName(presetName) + ?: throw ValidationException("Unknown parameters: $presetName") + if (overrides.isNotEmpty()) { + result = JsonOverrider.override(result, FullSeqAssemblerParameters::class.java, overrides) + ?: throw ValidationException("Failed to override some parameter: $overrides") + } + return result.withAllCoveredByFeature(targetFeature) + } + + override fun validate() { + ValidationException.requireNotEmpty(inputFiles) { "There is no files to process" } + inputFiles.forEach { input -> + ValidationException.requireFileType(input, InputFileType.CLNX) + } + if (!outputTemplate.endsWith(".clns")) { + throw ValidationException("Wrong template: command produces only clns, got $outputTemplate") + } + } + + override fun run1() { + val begin = Instant.now() + + val libraryRegistry = VDJCLibraryRegistry.getDefault() + val datasets = inputFiles.map { CloneSetIO.mkReader(it, libraryRegistry) } + ValidationException.require(datasets.all { it.header.allFullyCoveredBy != null }) { + val withoutFullyCovered = datasets.withIndex() + .filter { it.value.header.allFullyCoveredBy == null } + .map { inputFiles[it.index] } + .joinToString(", ") + "Some of the inputs were processed by `${CommandAssembleContigs.COMMAND_NAME}` or `${CommandAnalyze.COMMAND_NAME}` without `${AssembleContigsMixins.SetContigAssemblingFeatures.CMD_OPTION}` option: $withoutFullyCovered" + } + + ValidationException.require(datasets.all { it.header.allFullyCoveredBy != GeneFeatures(CDR3) }) { + val withoutCoveredCDR3 = datasets.withIndex() + .filter { it.value.header.allFullyCoveredBy == GeneFeatures(CDR3) } + .map { inputFiles[it.index] } + .joinToString(", ") + "Assemble feature must cover more than CDR3 in files: $withoutCoveredCDR3" + } + + ValidationException.require(datasets.all { it.header.allFullyCoveredBy!!.intersection(CDR3) != null }) { + val withoutCoveredCDR3 = datasets.withIndex() + .filter { it.value.header.allFullyCoveredBy!!.intersection(CDR3) == null } + .map { inputFiles[it.index] } + .joinToString(", ") + "Assemble feature must contain CDR3 in files: $withoutCoveredCDR3" + } + + val recalculateScores = GeneType.VJ_REFERENCE.any { geneType -> + val scores = datasets.map { + it.assemblerParameters.cloneFactoryParameters.getVJCParameters(geneType).scoring + } + scores.distinct().size != 1 + } + if (recalculateScores) { + logger.warn { + "Input files have different scoring for V or J genes. " + + "Hit scores will be recalculated for all clones, that will lead for loosing score info outside of assembling feature." + } + } + + val scoring = VJPair( + V = datasets.first().assemblerParameters.cloneFactoryParameters.vParameters, + J = datasets.first().assemblerParameters.cloneFactoryParameters.jParameters + ) + + ValidationException.requireTheSame(datasets.map { it.header.featuresToAlignMap }) { + "Require the same features to align for all input files" + } + + val targetFeature = (datasets.map { it.header.allFullyCoveredBy!! }.distinct() + maximumGeneFeature) + .reduce { acc, next -> + acc.intersection(next) ?: throw ValidationException("$acc and $next have no intersection") + } + + val assemblerParameters = assemblerParameters(targetFeature) + + datasets.forEachIndexed { i, reader -> + outputFiles[i].parent.createDirectories() + outputFiles[i].deleteIfExists() + if (!recalculateScores && reader.header.allFullyCoveredBy == targetFeature) { + logger.progress { "Copying ${inputFiles[i]}" } + inputFiles[i].copyTo(outputFiles[i]) + } else { + val cloneFactoryParameters = reader.assemblerParameters.cloneFactoryParameters + .setVParameters(scoring.V) + .setVParameters(scoring.J) + + val report: CommandReduceToCommonAssembleFeatureReport + val resultClones = if (reader.header.allFullyCoveredBy == targetFeature) { + TODO() + } else { + val cloneFactory = CloneFactory( + cloneFactoryParameters, + targetFeature.features.toTypedArray(), + reader.usedGenes, + reader.header.featuresToAlignMap + ) + + val reassembleReportBuilder = FullSeqAssemblerReportBuilder() + reassembleReportBuilder.setCommandLine(commandLineArguments) + reassembleReportBuilder.setInputFiles(inputFiles[i]) + reassembleReportBuilder.setOutputFiles(outputFiles[i]) + reassembleReportBuilder.setStartMillis(System.currentTimeMillis()) + + val clonesFilteredOutNoCDR3 = LongAdder() + val clonesFilteredOutFeatureIsNotAvailable = LongAdder() + + // TODO optional recalculation of the score + val result = reader.readClones() + .reportProgress("Grouping by assemble feature: ${inputFiles[i]}") + .filter { clone -> + if (targetFeature.features.any { !clone.isAvailable(it) }) { + // non-functional clones that don't have a reference point for the target feature + clonesFilteredOutFeatureIsNotAvailable.increment() + false + } else if (!clone.isAvailable(CDR3)) { + clonesFilteredOutNoCDR3.increment() + false + } else + true + } + .groupByOnDisk( + ComparatorWithHash.Companion.compareBy { clone -> + targetFeature.features.map { clone.getNFeature(it)!! } + .reduce { acc, next -> acc + next } + }, + tempDest, + "group_by_assemble_feature", + stateBuilder = reader.header.constructStateBuilder(reader.usedGenes) + ) + .reportProgress("Reassembling ${inputFiles[i]}") + .map { it.toList() } + // also make `it.toList()` synchronized + .buffered(threadsOptions.value) + .mapInParallel(1) { clones -> + val clone = clones.minBy { it.ranks.byReads }.cutByCDR3() + val fullSeqAssembler = FullSeqAssembler( + cloneFactory, assemblerParameters, + arrayOf(CDR3), + clone, reader.header.alignerParameters, + clone.getBestHit(Variable), clone.getBestHit(Joining) + ) + fullSeqAssembler.report = reassembleReportBuilder + val rawVariantsData = fullSeqAssembler.calculateRawData { + CUtils.asOutputPort(clones + .flatMap { clone -> + clone.tagCount.tuples().map { tuple -> + VDJCAlignments( + hits = clone.hits, + tagCount = TagCount(tuple), + targets = clone.getTargets(), + history = null, + originalSequences = null + ) + } + } + ) + } + fullSeqAssembler.callVariants(rawVariantsData) + } + .toList() + .flatMap { it.toList() } + report = CommandReduceToCommonAssembleFeatureReport.Reassemble( + reassembleReportBuilder.buildReport(), + clonesFilteredOutNoCDR3 = clonesFilteredOutNoCDR3.sum(), + clonesFilteredOutFeatureIsNotAvailable = clonesFilteredOutFeatureIsNotAvailable.sum() + ) + result + } + val resultHeader = reader.header + .copy(allFullyCoveredBy = targetFeature) + .copy( + assemblerParameters = reader.header.assemblerParameters!! + .setCloneFactoryParameters(cloneFactoryParameters) + ) + + val cloneSet = CloneSet.Builder(resultClones, reader.usedGenes, resultHeader) + .sort(reader.ordering) + .recalculateRanks() + .calculateTotalCounts() + .build() + ClnsWriter(outputFiles[i]).use { writer -> + writer.writeCloneSet(cloneSet) + writer.setFooter( + reader.footer.addStepReport( + MiXCRCommandDescriptor.reduceToCommonAssembleFeature, + report + ) + ) + } + // Writing report to stout + ReportUtil.writeReportToStdout(report) + reportOptions.appendToFiles(report) + } + } + + logger.progress { + "Analysis time: " + FormatUtils.nanoTimeToString(Duration.between(begin, Instant.now()).toNanos()) + } + } + + private fun Clone.cutByCDR3(): Clone { + // TODO do it in one go + val cutHits = EnumMap>(GeneType::class.java) + for (geneType in arrayOf(Diversity, Constant)) { + if (getBestHit(geneType) != null) + cutHits[geneType] = getHits(geneType) + } + for (geneType in GeneType.VJ_REFERENCE) { + val converted = getHits(geneType).map { hit -> + hit.mapAlignments { alignment -> + if (geneType == Variable) { + val from = hit.gene.partitioning.getRelativePosition(hit.alignedFeature, CDR3Begin) + if (from !in alignment.sequence1Range) return@mapAlignments null + val rangeToKeep = alignment.sequence1Range.setLower(from) + val resultMutations = alignment.absoluteMutations.extractAbsoluteMutationsForRange(rangeToKeep) + val seq2Range = alignment.sequence2Range.setLower( + alignment.sequence2Range.to - rangeToKeep.length() - resultMutations.lengthDelta + ) + Alignment( + alignment.sequence1, + resultMutations, + rangeToKeep, + seq2Range, + alignment.score + ) + } else { + val to = hit.gene.partitioning.getRelativePosition(hit.alignedFeature, CDR3End) + if (to !in alignment.sequence1Range) return@mapAlignments null + val rangeToKeep = alignment.sequence1Range.setUpper(to) + val resultMutations = alignment.absoluteMutations.extractAbsoluteMutationsForRange(rangeToKeep) + val seq2Range = alignment.sequence2Range.setUpper( + alignment.sequence2Range.from + rangeToKeep.length() + resultMutations.lengthDelta + ) + Alignment( + alignment.sequence1, + resultMutations, + rangeToKeep, + seq2Range, + alignment.score + ) + } + } + } + cutHits[geneType] = converted.toTypedArray() + } + val withCutCDR3 = withHits(cutHits) + val shift = withCutCDR3.getBestHit(Variable)!!.getAlignment(0)!!.sequence2Range.from + val shiftedHits = EnumMap>(GeneType::class.java) + withCutCDR3.hits.map { (geneType, hits) -> + shiftedHits[geneType] = hits.map { hit -> + hit.mapAlignments { alignment -> + alignment.move(-shift) + } + }.toTypedArray() + } + return withCutCDR3 + .withHits(shiftedHits) + .withTargets(arrayOf(getFeature(CDR3)!!)) + } + + companion object { + const val COMMAND_NAME = MiXCRCommandDescriptor.reduceToCommonAssembleFeature.name + + private const val inputsLabel = "(input_file.(clns|clna)|directory)..." + + private const val outputLabel = "template.clns" + + + fun mkCommandSpec(): CommandSpec = + CommandSpec.forAnnotatedObject(CommandReduceToCommonAssembleFeature::class.java) + .addPositional( + CommandLine.Model.PositionalParamSpec.builder() + .index("0") + .required(false) + .arity("0..*") + .type(Path::class.java) + .paramLabel(inputsLabel) + .hideParamSyntax(true) + .description( + "Input files or directory with files for process.", + "In case of directory no filter by file type will be applied." + ) + .build() + ) + .addPositional( + CommandLine.Model.PositionalParamSpec.builder() + .index("1") + .required(false) + .arity("0..*") + .type(Path::class.java) + .paramLabel(outputLabel) + .hideParamSyntax(true) + .description(OutputTemplate.description) + .build() + ) + } +} diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandRefineTagsAndSort.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandRefineTagsAndSort.kt index 4f2d3b0a7..e708dc8aa 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandRefineTagsAndSort.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandRefineTagsAndSort.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -39,7 +39,7 @@ import com.milaboratory.mixcr.basictypes.tag.TagValueType import com.milaboratory.mixcr.basictypes.tag.tagAliases import com.milaboratory.mixcr.cli.CommonDescriptions.DEFAULT_VALUE_FROM_PRESET import com.milaboratory.mixcr.cli.MiXCRMixinCollection.Companion.mixins -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.RefineTagsAndSortMixins import com.milaboratory.mixcr.util.MiXCRVersionInfo @@ -61,7 +61,7 @@ import java.nio.file.Path import java.util.* object CommandRefineTagsAndSort { - const val COMMAND_NAME = MiXCRCommandDescriptor.refineTagsAndSort.name + const val COMMAND_NAME = AnalyzeCommandDescriptor.refineTagsAndSort.name abstract class CmdBase : MiXCRCommandWithOutputs(), MiXCRPresetAwareCommand { @Option( @@ -436,7 +436,7 @@ object CommandRefineTagsAndSort { writer.writeHeader( header .updateTagInfo { tagsInfo -> tagsInfo.setSorted(tagsInfo.size) } - .addStepParams(MiXCRCommandDescriptor.refineTagsAndSort, cmdParams) + .addStepParams(AnalyzeCommandDescriptor.refineTagsAndSort, cmdParams) .copy(paramsSpec = dontSavePresetOption.presetToSave(paramsSpec)), mainReader.usedGenes ) @@ -454,7 +454,7 @@ object CommandRefineTagsAndSort { writer.setFooter( mainReader.footer .withThresholds(thresholds) - .addStepReport(MiXCRCommandDescriptor.refineTagsAndSort, refineTagsAndSortReport) + .addStepReport(AnalyzeCommandDescriptor.refineTagsAndSort, refineTagsAndSortReport) ) } } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt index b32b08424..0791cdc07 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt @@ -221,6 +221,10 @@ object Main { ) .commandsGroup( CommandsGroup("Util commands") + .addSubcommand( + CommandReduceToCommonAssembleFeature.COMMAND_NAME, + CommandReduceToCommonAssembleFeature::class.java + ) .addSubcommand("exportReadsForClones", CommandExportReadsForClones::class.java) .addSubcommand("exportAlignmentsForClones", CommandExportAlignmentsForClones::class.java) .addSubcommand("exportReads", CommandExportReads::class.java) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/MiXCRParamsResolver.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/MiXCRParamsResolver.kt index f00c0f8c6..84f408345 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/MiXCRParamsResolver.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/MiXCRParamsResolver.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -17,8 +17,8 @@ import com.milaboratory.cli.PresetAware import com.milaboratory.mixcr.basictypes.HasFeatureToAlign import com.milaboratory.mixcr.cli.CommonDescriptions.Labels import com.milaboratory.mixcr.presets.AlignMixins +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.Flags -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRMixin import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.Presets @@ -39,18 +39,18 @@ abstract class MiXCRParamsResolver

( throw ValidationException("Error validating preset bundle."); } if ( - bundle.pipeline?.steps?.contains(MiXCRCommandDescriptor.assembleContigs) == true && + bundle.pipeline?.steps?.contains(AnalyzeCommandDescriptor.assembleContigs) == true && bundle.assemble?.clnaOutput == false ) throw ValidationException("assembleContigs step required clnaOutput=true on assemble step") bundle.align?.parameters?.featuresToAlignMap?.let { HasFeatureToAlign(it) }?.let { featuresToAlign -> - if (bundle.pipeline?.steps?.contains(MiXCRCommandDescriptor.assemble) == true) { + if (bundle.pipeline?.steps?.contains(AnalyzeCommandDescriptor.assemble) == true) { bundle.assemble?.let { params -> CommandAssemble.validateParams(params, featuresToAlign) } } - if (bundle.pipeline?.steps?.contains(MiXCRCommandDescriptor.assembleContigs) == true) { + if (bundle.pipeline?.steps?.contains(AnalyzeCommandDescriptor.assembleContigs) == true) { bundle.assembleContigs?.let { params -> CommandAssembleContigs.validateParams(params, featuresToAlign) } diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/Mixins.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/Mixins.kt index 3141107a8..b2eb64075 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/Mixins.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/Mixins.kt @@ -33,6 +33,7 @@ import com.milaboratory.mixcr.presets.AlignMixins.RightAlignmentBoundaryWithPoin import com.milaboratory.mixcr.presets.AlignMixins.SetLibrary import com.milaboratory.mixcr.presets.AlignMixins.SetSpecies import com.milaboratory.mixcr.presets.AlignMixins.SetTagPattern +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.AssembleContigsMixins.SetContigAssemblingFeatures import com.milaboratory.mixcr.presets.AssembleMixins.SetClonotypeAssemblingFeatures import com.milaboratory.mixcr.presets.AssembleMixins.SetSplitClonesBy @@ -43,7 +44,6 @@ import com.milaboratory.mixcr.presets.ExportMixins.AddExportClonesField import com.milaboratory.mixcr.presets.ExportMixins.DontImputeGermlineOnExport import com.milaboratory.mixcr.presets.ExportMixins.ImputeGermlineOnExport import com.milaboratory.mixcr.presets.GenericMixin -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.PipelineMixins.AddPipelineStep import com.milaboratory.mixcr.presets.PipelineMixins.RemovePipelineStep import com.milaboratory.mixcr.presets.QcMixins @@ -699,9 +699,9 @@ object ExportMiXCRMixins { @Option( description = [ "Filter out clones from groups of particular type.", - "`found` - groups that were found on `${MiXCRCommandDescriptor.assembleCells.name}`.", - "`undefined` - there were not enough info on `${MiXCRCommandDescriptor.assembleCells.name}` to form a group.", - "`contamination` - clones that were marked as contamination on `${MiXCRCommandDescriptor.assembleCells.name}`.", + "`found` - groups that were found on `${AnalyzeCommandDescriptor.assembleCells.name}`.", + "`undefined` - there were not enough info on `${AnalyzeCommandDescriptor.assembleCells.name}` to form a group.", + "`contamination` - clones that were marked as contamination on `${AnalyzeCommandDescriptor.assembleCells.name}`.", ], names = [ExportMixins.FilterOutCloneGroupTypes.CMD_OPTION], arity = "1..*", diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/OutputTemplate.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/OutputTemplate.kt new file mode 100644 index 000000000..0231cbdd0 --- /dev/null +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/OutputTemplate.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved + * + * Before downloading or accessing the software, please read carefully the + * License Agreement available at: + * https://github.com/milaboratory/mixcr/blob/develop/LICENSE + * + * By downloading or accessing the software, you accept and agree to be bound + * by the terms of the License Agreement. If you do not want to agree to the terms + * of the Licensing Agreement, you must not download or access the software. + */ +package com.milaboratory.mixcr.cli + +import com.milaboratory.app.ValidationException +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.nameWithoutExtension + +object OutputTemplate { + const val description = "Output template may contain {file_name} and {file_dir_path},%n" + + "outputs for '-o /output/folder/{file_name}_suffix.clns input_file.clns input_file2.clns' will be /output/folder/input_file_suffix.clns and /output/folder/input_file2_suffix.clns,%n" + + "outputs for '-o {file_dir_path}/{file_name}_suffix.clns /some/folder1/input_file.clns /some/folder2/input_file2.clns' will be /seme/folder1/input_file_suffix.clns and /some/folder2/input_file2_suffix.clns%n" + + "Resulted outputs must be uniq" + + fun calculateOutputs(template: String, inputFiles: List): List { + val outputFiles = inputFiles + .map { it.toAbsolutePath() } + .map { path -> + template + .replace(Regex("\\{file_name}"), path.nameWithoutExtension) + .replace(Regex("\\{file_dir_path}"), path.parent.toString()) + } + .map { Paths.get(it) } + .toList() + if (outputFiles.distinct().count() < outputFiles.size) { + var message = "Output files are not uniq: $outputFiles" + message += "\nTry to use `{file_name}` and/or `{file_dir_path}` in template to get different output paths for every input. See help for more details" + throw ValidationException(message) + } + return outputFiles + } +} diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/qc/CommandExportQcTags.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/qc/CommandExportQcTags.kt index a260e802d..0d4c271e5 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/qc/CommandExportQcTags.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/qc/CommandExportQcTags.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -21,7 +21,7 @@ import com.milaboratory.mixcr.cli.CommandRefineTagsAndSort import com.milaboratory.mixcr.cli.CommonDescriptions.Labels import com.milaboratory.mixcr.cli.MiXCRCommandWithOutputs import com.milaboratory.mixcr.cli.exportTypes -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.qc.plots.TagRefinementQc import picocli.CommandLine import picocli.CommandLine.Command @@ -101,7 +101,7 @@ class CommandExportQcTags : MiXCRCommandWithOutputs() { override fun run1() { val plots = inputFiles.mapNotNull { file -> val info = IOUtil.extractFileInfo(file) - val report = info.footer.reports[MiXCRCommandDescriptor.refineTagsAndSort] + val report = info.footer.reports[AnalyzeCommandDescriptor.refineTagsAndSort] if (report.isEmpty()) { println("No tag refinement report for $file; did you run ${CommandRefineTagsAndSort.COMMAND_NAME} command?") null diff --git a/src/test/kotlin/com/milaboratory/mixcr/PresetsTest.kt b/src/test/kotlin/com/milaboratory/mixcr/PresetsTest.kt index 874229477..07660e233 100644 --- a/src/test/kotlin/com/milaboratory/mixcr/PresetsTest.kt +++ b/src/test/kotlin/com/milaboratory/mixcr/PresetsTest.kt @@ -11,7 +11,7 @@ import com.milaboratory.mixcr.cli.presetFlagsMessages import com.milaboratory.mixcr.export.CloneFieldsExtractorsFactory import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.VDJCAlignmentsFieldsExtractorsFactory -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.mixcr.presets.MiXCRPresetCategory import com.milaboratory.mixcr.presets.Presets @@ -152,7 +152,7 @@ class PresetsTest { fun `all presets should have exportClones step`() { Presets.nonAbstractPresetNames.filter { presetName -> val bundle = Presets.MiXCRBundleResolver.resolvePreset(presetName) - MiXCRCommandDescriptor.exportClones !in bundle.pipeline!!.steps + AnalyzeCommandDescriptor.exportClones !in bundle.pipeline!!.steps } shouldBe emptyList() } @@ -162,7 +162,7 @@ class PresetsTest { presetName.asClue { val bundle = Presets.MiXCRBundleResolver.resolvePreset(presetName) bundle.pipeline!!.steps - .filterIsInstance>() + .filterIsInstance>() .forEach { exportStep -> exportStep.command.asClue { bundle.pipeline!!.steps shouldContainAnyOf exportStep.runAfterLastOf() @@ -177,11 +177,11 @@ class PresetsTest { Presets.nonAbstractPresetNames .filter { presetName -> val bundle = Presets.MiXCRBundleResolver.resolvePreset(presetName) - MiXCRCommandDescriptor.assembleCells in bundle.pipeline!!.steps + AnalyzeCommandDescriptor.assembleCells in bundle.pipeline!!.steps } .filter { presetName -> val bundle = Presets.MiXCRBundleResolver.resolvePreset(presetName) - MiXCRCommandDescriptor.exportCloneGroups !in bundle.pipeline!!.steps + AnalyzeCommandDescriptor.exportCloneGroups !in bundle.pipeline!!.steps } shouldBe emptyList() } diff --git a/src/test/kotlin/com/milaboratory/mixcr/cli/CommandExportPresetTest.kt b/src/test/kotlin/com/milaboratory/mixcr/cli/CommandExportPresetTest.kt index f689c2673..139dfb81b 100644 --- a/src/test/kotlin/com/milaboratory/mixcr/cli/CommandExportPresetTest.kt +++ b/src/test/kotlin/com/milaboratory/mixcr/cli/CommandExportPresetTest.kt @@ -2,7 +2,7 @@ package com.milaboratory.mixcr.cli import com.fasterxml.jackson.module.kotlin.readValue import com.milaboratory.mixcr.export.ExportFieldDescription -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRParamsBundle import com.milaboratory.util.K_YAML_OM import com.milaboratory.util.TempFileManager @@ -52,7 +52,7 @@ class CommandExportPresetTest { output.delete() TestMain.execute("exportPreset --species hs --floating-left-alignment-boundary --floating-right-alignment-boundary C --dna --add-step assembleContigs --preset-name test-tcr-shotgun ${output.path}") val result = K_YAML_OM.readValue(output) - result.pipeline!!.steps shouldContain MiXCRCommandDescriptor.assembleContigs + result.pipeline!!.steps shouldContain AnalyzeCommandDescriptor.assembleContigs result.assemble!!.clnaOutput shouldBe true } @@ -62,8 +62,8 @@ class CommandExportPresetTest { output.delete() TestMain.execute("exportPreset --floating-left-alignment-boundary --floating-right-alignment-boundary C --species hs --dna --remove-step exportClones --remove-step exportAlignments --preset-name test-tcr-shotgun ${output.path}") val result = K_YAML_OM.readValue(output) - result.pipeline!!.steps shouldNotContain MiXCRCommandDescriptor.exportClones - result.pipeline!!.steps shouldNotContain MiXCRCommandDescriptor.exportAlignments + result.pipeline!!.steps shouldNotContain AnalyzeCommandDescriptor.exportClones + result.pipeline!!.steps shouldNotContain AnalyzeCommandDescriptor.exportAlignments } @Test @@ -85,8 +85,8 @@ class CommandExportPresetTest { val result = K_YAML_OM.readValue(output) result.flags shouldBe emptySet() result.align!!.species shouldBe "hsa" - result.pipeline!!.steps shouldNotContain MiXCRCommandDescriptor.exportClones - result.pipeline!!.steps shouldNotContain MiXCRCommandDescriptor.exportAlignments + result.pipeline!!.steps shouldNotContain AnalyzeCommandDescriptor.exportClones + result.pipeline!!.steps shouldNotContain AnalyzeCommandDescriptor.exportAlignments result.align!!.tagPattern shouldBe "^(R1F:N{0:2}(C:gggggaaaagggttg)(R1:*))" result.align!!.tagUnstranded shouldBe true result.align!!.limit shouldBe 1000 diff --git a/src/test/kotlin/com/milaboratory/mixcr/cli/MetaInfoTest.kt b/src/test/kotlin/com/milaboratory/mixcr/cli/MetaInfoTest.kt index a5fd5f709..2501c18f1 100644 --- a/src/test/kotlin/com/milaboratory/mixcr/cli/MetaInfoTest.kt +++ b/src/test/kotlin/com/milaboratory/mixcr/cli/MetaInfoTest.kt @@ -1,5 +1,6 @@ package com.milaboratory.mixcr.cli +import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import io.kotest.assertions.withClue import io.kotest.matchers.shouldBe @@ -13,7 +14,7 @@ class MetaInfoTest { mixcrCommandDescriptions() .map { it.command } .filter { commandName -> - MiXCRCommandDescriptor.fromStringOrNull(commandName) == null + AnalyzeCommandDescriptor.fromStringOrNull(commandName) == null } shouldBe emptyList() } @@ -24,7 +25,7 @@ class MetaInfoTest { MiXCRCommandDescriptor.findAlleles, MiXCRCommandDescriptor.findShmTrees, - MiXCRCommandDescriptor.align + AnalyzeCommandDescriptor.align ) (mixcrCommandDescriptions() - exclusions) .map { it.command } @@ -41,10 +42,10 @@ class MetaInfoTest { MiXCRCommandDescriptor.findAlleles, MiXCRCommandDescriptor.findShmTrees, - MiXCRCommandDescriptor.exportClones, - MiXCRCommandDescriptor.exportCloneGroups, - MiXCRCommandDescriptor.exportAlignments, - MiXCRCommandDescriptor.qc, + AnalyzeCommandDescriptor.exportClones, + AnalyzeCommandDescriptor.exportCloneGroups, + AnalyzeCommandDescriptor.exportAlignments, + AnalyzeCommandDescriptor.qc, ) (mixcrCommandDescriptions() - exclusions) .map { it.command } @@ -55,7 +56,7 @@ class MetaInfoTest { } private fun mixcrCommandDescriptions() = - MiXCRCommandDescriptor::class.resolveSealedSubclasses().map { it.objectInstance!! } + AnalyzeCommandDescriptor::class.resolveSealedSubclasses().map { it.objectInstance!! } private fun KClass.resolveSealedSubclasses(): List> = when { From cd992ce3ec8ad7be6d00e34daafe66eba471a705 Mon Sep 17 00:00:00 2001 From: gnefedev Date: Thu, 15 Feb 2024 12:12:32 +0100 Subject: [PATCH 2/6] remove CommandReduceToCommonAssembleFeature --- .../CommandReduceToCommonAssembleFeature.kt | 440 ------------------ .../kotlin/com/milaboratory/mixcr/cli/Main.kt | 4 - 2 files changed, 444 deletions(-) delete mode 100644 src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt deleted file mode 100644 index 0361aa007..000000000 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandReduceToCommonAssembleFeature.kt +++ /dev/null @@ -1,440 +0,0 @@ -/* - * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved - * - * Before downloading or accessing the software, please read carefully the - * License Agreement available at: - * https://github.com/milaboratory/mixcr/blob/develop/LICENSE - * - * By downloading or accessing the software, you accept and agree to be bound - * by the terms of the License Agreement. If you do not want to agree to the terms - * of the Licensing Agreement, you must not download or access the software. - */ -package com.milaboratory.mixcr.cli - -import cc.redberry.pipe.CUtils -import cc.redberry.pipe.util.buffered -import cc.redberry.pipe.util.filter -import cc.redberry.pipe.util.map -import cc.redberry.pipe.util.mapInParallel -import cc.redberry.pipe.util.toList -import com.milaboratory.app.InputFileType -import com.milaboratory.app.ValidationException -import com.milaboratory.app.logger -import com.milaboratory.core.alignment.Alignment -import com.milaboratory.mixcr.assembler.CloneFactory -import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssembler -import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerParameters -import com.milaboratory.mixcr.assembler.fullseq.FullSeqAssemblerReportBuilder -import com.milaboratory.mixcr.basictypes.ClnsWriter -import com.milaboratory.mixcr.basictypes.Clone -import com.milaboratory.mixcr.basictypes.CloneSet -import com.milaboratory.mixcr.basictypes.CloneSetIO -import com.milaboratory.mixcr.basictypes.VDJCAlignments -import com.milaboratory.mixcr.basictypes.VDJCHit -import com.milaboratory.mixcr.basictypes.tag.TagCount -import com.milaboratory.mixcr.cli.CommonDescriptions.Labels -import com.milaboratory.mixcr.presets.AssembleContigsMixins -import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor -import com.milaboratory.mixcr.trees.constructStateBuilder -import com.milaboratory.mixcr.util.VJPair -import com.milaboratory.mixcr.util.plus -import com.milaboratory.util.ComparatorWithHash -import com.milaboratory.util.FormatUtils -import com.milaboratory.util.JsonOverrider -import com.milaboratory.util.ReportUtil -import com.milaboratory.util.TempFileDest -import com.milaboratory.util.TempFileManager -import com.milaboratory.util.groupByOnDisk -import io.repseq.core.GeneFeature.CDR3 -import io.repseq.core.GeneFeature.VDJRegion -import io.repseq.core.GeneFeatures -import io.repseq.core.GeneType -import io.repseq.core.GeneType.Constant -import io.repseq.core.GeneType.Diversity -import io.repseq.core.GeneType.Joining -import io.repseq.core.GeneType.Variable -import io.repseq.core.ReferencePoint.CDR3Begin -import io.repseq.core.ReferencePoint.CDR3End -import io.repseq.core.VDJCLibraryRegistry -import picocli.CommandLine -import picocli.CommandLine.Command -import picocli.CommandLine.Help.Visibility.ALWAYS -import picocli.CommandLine.Mixin -import picocli.CommandLine.Model.CommandSpec -import picocli.CommandLine.Option -import picocli.CommandLine.Parameters -import java.nio.file.Path -import java.nio.file.Paths -import java.time.Duration -import java.time.Instant -import java.util.* -import java.util.concurrent.atomic.LongAdder -import kotlin.io.path.copyTo -import kotlin.io.path.createDirectories -import kotlin.io.path.deleteIfExists -import kotlin.io.path.isDirectory -import kotlin.io.path.listDirectoryEntries - -@Command( - description = [ - "Reassemble clones so, that all outputs will have the same assemble feature, thus could be used for inferring alleles and building SHM trees." - ] -) -class CommandReduceToCommonAssembleFeature : MiXCRCommandWithOutputs() { - @Parameters( - index = "0", - arity = "2..*", - paramLabel = "$inputsLabel $outputLabel", - hideParamSyntax = true, - // help is covered by mkCommandSpec - hidden = true - ) - val inOut: List = mutableListOf() - - @Option( - names = ["--maximum-gene-feature"], - required = false, - paramLabel = Labels.GENE_FEATURE, - description = ["Limit result intersected feature by this."], - showDefaultValue = ALWAYS - ) - var maximumGeneFeature: GeneFeatures = GeneFeatures(VDJRegion) - - @Mixin - lateinit var threadsOptions: ThreadsOption - - @Mixin - lateinit var useLocalTemp: UseLocalTempOption - - @Mixin - lateinit var reportOptions: ReportOptions - - @Option( - names = ["-O"], - description = ["Overrides default find alleles parameter values"], - paramLabel = Labels.OVERRIDES, - order = OptionsOrder.overrides - ) - var overrides: Map = mutableMapOf() - - private val outputTemplate get() = inOut.last() - - override val inputFiles: List - get() = inOut.dropLast(1).map { Paths.get(it) }.flatMap { path -> - when { - path.isDirectory() -> path.listDirectoryEntries() - else -> listOf(path) - } - } - - override val outputFiles: List by lazy { - OutputTemplate.calculateOutputs(outputTemplate, inputFiles) - } - - private val tempDest: TempFileDest by lazy { - val path = outputFiles.first() - if (useLocalTemp.value) path.toAbsolutePath().parent.createDirectories() - TempFileManager.smartTempDestination(path, ".cutToCommonFeature", !useLocalTemp.value) - } - - private fun assemblerParameters(targetFeature: GeneFeatures): FullSeqAssemblerParameters { - val presetName = "default" - var result = FullSeqAssemblerParameters.presets.getByName(presetName) - ?: throw ValidationException("Unknown parameters: $presetName") - if (overrides.isNotEmpty()) { - result = JsonOverrider.override(result, FullSeqAssemblerParameters::class.java, overrides) - ?: throw ValidationException("Failed to override some parameter: $overrides") - } - return result.withAllCoveredByFeature(targetFeature) - } - - override fun validate() { - ValidationException.requireNotEmpty(inputFiles) { "There is no files to process" } - inputFiles.forEach { input -> - ValidationException.requireFileType(input, InputFileType.CLNX) - } - if (!outputTemplate.endsWith(".clns")) { - throw ValidationException("Wrong template: command produces only clns, got $outputTemplate") - } - } - - override fun run1() { - val begin = Instant.now() - - val libraryRegistry = VDJCLibraryRegistry.getDefault() - val datasets = inputFiles.map { CloneSetIO.mkReader(it, libraryRegistry) } - ValidationException.require(datasets.all { it.header.allFullyCoveredBy != null }) { - val withoutFullyCovered = datasets.withIndex() - .filter { it.value.header.allFullyCoveredBy == null } - .map { inputFiles[it.index] } - .joinToString(", ") - "Some of the inputs were processed by `${CommandAssembleContigs.COMMAND_NAME}` or `${CommandAnalyze.COMMAND_NAME}` without `${AssembleContigsMixins.SetContigAssemblingFeatures.CMD_OPTION}` option: $withoutFullyCovered" - } - - ValidationException.require(datasets.all { it.header.allFullyCoveredBy != GeneFeatures(CDR3) }) { - val withoutCoveredCDR3 = datasets.withIndex() - .filter { it.value.header.allFullyCoveredBy == GeneFeatures(CDR3) } - .map { inputFiles[it.index] } - .joinToString(", ") - "Assemble feature must cover more than CDR3 in files: $withoutCoveredCDR3" - } - - ValidationException.require(datasets.all { it.header.allFullyCoveredBy!!.intersection(CDR3) != null }) { - val withoutCoveredCDR3 = datasets.withIndex() - .filter { it.value.header.allFullyCoveredBy!!.intersection(CDR3) == null } - .map { inputFiles[it.index] } - .joinToString(", ") - "Assemble feature must contain CDR3 in files: $withoutCoveredCDR3" - } - - val recalculateScores = GeneType.VJ_REFERENCE.any { geneType -> - val scores = datasets.map { - it.assemblerParameters.cloneFactoryParameters.getVJCParameters(geneType).scoring - } - scores.distinct().size != 1 - } - if (recalculateScores) { - logger.warn { - "Input files have different scoring for V or J genes. " + - "Hit scores will be recalculated for all clones, that will lead for loosing score info outside of assembling feature." - } - } - - val scoring = VJPair( - V = datasets.first().assemblerParameters.cloneFactoryParameters.vParameters, - J = datasets.first().assemblerParameters.cloneFactoryParameters.jParameters - ) - - ValidationException.requireTheSame(datasets.map { it.header.featuresToAlignMap }) { - "Require the same features to align for all input files" - } - - val targetFeature = (datasets.map { it.header.allFullyCoveredBy!! }.distinct() + maximumGeneFeature) - .reduce { acc, next -> - acc.intersection(next) ?: throw ValidationException("$acc and $next have no intersection") - } - - val assemblerParameters = assemblerParameters(targetFeature) - - datasets.forEachIndexed { i, reader -> - outputFiles[i].parent.createDirectories() - outputFiles[i].deleteIfExists() - if (!recalculateScores && reader.header.allFullyCoveredBy == targetFeature) { - logger.progress { "Copying ${inputFiles[i]}" } - inputFiles[i].copyTo(outputFiles[i]) - } else { - val cloneFactoryParameters = reader.assemblerParameters.cloneFactoryParameters - .setVParameters(scoring.V) - .setVParameters(scoring.J) - - val report: CommandReduceToCommonAssembleFeatureReport - val resultClones = if (reader.header.allFullyCoveredBy == targetFeature) { - TODO() - } else { - val cloneFactory = CloneFactory( - cloneFactoryParameters, - targetFeature.features.toTypedArray(), - reader.usedGenes, - reader.header.featuresToAlignMap - ) - - val reassembleReportBuilder = FullSeqAssemblerReportBuilder() - reassembleReportBuilder.setCommandLine(commandLineArguments) - reassembleReportBuilder.setInputFiles(inputFiles[i]) - reassembleReportBuilder.setOutputFiles(outputFiles[i]) - reassembleReportBuilder.setStartMillis(System.currentTimeMillis()) - - val clonesFilteredOutNoCDR3 = LongAdder() - val clonesFilteredOutFeatureIsNotAvailable = LongAdder() - - // TODO optional recalculation of the score - val result = reader.readClones() - .reportProgress("Grouping by assemble feature: ${inputFiles[i]}") - .filter { clone -> - if (targetFeature.features.any { !clone.isAvailable(it) }) { - // non-functional clones that don't have a reference point for the target feature - clonesFilteredOutFeatureIsNotAvailable.increment() - false - } else if (!clone.isAvailable(CDR3)) { - clonesFilteredOutNoCDR3.increment() - false - } else - true - } - .groupByOnDisk( - ComparatorWithHash.Companion.compareBy { clone -> - targetFeature.features.map { clone.getNFeature(it)!! } - .reduce { acc, next -> acc + next } - }, - tempDest, - "group_by_assemble_feature", - stateBuilder = reader.header.constructStateBuilder(reader.usedGenes) - ) - .reportProgress("Reassembling ${inputFiles[i]}") - .map { it.toList() } - // also make `it.toList()` synchronized - .buffered(threadsOptions.value) - .mapInParallel(1) { clones -> - val clone = clones.minBy { it.ranks.byReads }.cutByCDR3() - val fullSeqAssembler = FullSeqAssembler( - cloneFactory, assemblerParameters, - arrayOf(CDR3), - clone, reader.header.alignerParameters, - clone.getBestHit(Variable), clone.getBestHit(Joining) - ) - fullSeqAssembler.report = reassembleReportBuilder - val rawVariantsData = fullSeqAssembler.calculateRawData { - CUtils.asOutputPort(clones - .flatMap { clone -> - clone.tagCount.tuples().map { tuple -> - VDJCAlignments( - hits = clone.hits, - tagCount = TagCount(tuple), - targets = clone.getTargets(), - history = null, - originalSequences = null - ) - } - } - ) - } - fullSeqAssembler.callVariants(rawVariantsData) - } - .toList() - .flatMap { it.toList() } - report = CommandReduceToCommonAssembleFeatureReport.Reassemble( - reassembleReportBuilder.buildReport(), - clonesFilteredOutNoCDR3 = clonesFilteredOutNoCDR3.sum(), - clonesFilteredOutFeatureIsNotAvailable = clonesFilteredOutFeatureIsNotAvailable.sum() - ) - result - } - val resultHeader = reader.header - .copy(allFullyCoveredBy = targetFeature) - .copy( - assemblerParameters = reader.header.assemblerParameters!! - .setCloneFactoryParameters(cloneFactoryParameters) - ) - - val cloneSet = CloneSet.Builder(resultClones, reader.usedGenes, resultHeader) - .sort(reader.ordering) - .recalculateRanks() - .calculateTotalCounts() - .build() - ClnsWriter(outputFiles[i]).use { writer -> - writer.writeCloneSet(cloneSet) - writer.setFooter( - reader.footer.addStepReport( - MiXCRCommandDescriptor.reduceToCommonAssembleFeature, - report - ) - ) - } - // Writing report to stout - ReportUtil.writeReportToStdout(report) - reportOptions.appendToFiles(report) - } - } - - logger.progress { - "Analysis time: " + FormatUtils.nanoTimeToString(Duration.between(begin, Instant.now()).toNanos()) - } - } - - private fun Clone.cutByCDR3(): Clone { - // TODO do it in one go - val cutHits = EnumMap>(GeneType::class.java) - for (geneType in arrayOf(Diversity, Constant)) { - if (getBestHit(geneType) != null) - cutHits[geneType] = getHits(geneType) - } - for (geneType in GeneType.VJ_REFERENCE) { - val converted = getHits(geneType).map { hit -> - hit.mapAlignments { alignment -> - if (geneType == Variable) { - val from = hit.gene.partitioning.getRelativePosition(hit.alignedFeature, CDR3Begin) - if (from !in alignment.sequence1Range) return@mapAlignments null - val rangeToKeep = alignment.sequence1Range.setLower(from) - val resultMutations = alignment.absoluteMutations.extractAbsoluteMutationsForRange(rangeToKeep) - val seq2Range = alignment.sequence2Range.setLower( - alignment.sequence2Range.to - rangeToKeep.length() - resultMutations.lengthDelta - ) - Alignment( - alignment.sequence1, - resultMutations, - rangeToKeep, - seq2Range, - alignment.score - ) - } else { - val to = hit.gene.partitioning.getRelativePosition(hit.alignedFeature, CDR3End) - if (to !in alignment.sequence1Range) return@mapAlignments null - val rangeToKeep = alignment.sequence1Range.setUpper(to) - val resultMutations = alignment.absoluteMutations.extractAbsoluteMutationsForRange(rangeToKeep) - val seq2Range = alignment.sequence2Range.setUpper( - alignment.sequence2Range.from + rangeToKeep.length() + resultMutations.lengthDelta - ) - Alignment( - alignment.sequence1, - resultMutations, - rangeToKeep, - seq2Range, - alignment.score - ) - } - } - } - cutHits[geneType] = converted.toTypedArray() - } - val withCutCDR3 = withHits(cutHits) - val shift = withCutCDR3.getBestHit(Variable)!!.getAlignment(0)!!.sequence2Range.from - val shiftedHits = EnumMap>(GeneType::class.java) - withCutCDR3.hits.map { (geneType, hits) -> - shiftedHits[geneType] = hits.map { hit -> - hit.mapAlignments { alignment -> - alignment.move(-shift) - } - }.toTypedArray() - } - return withCutCDR3 - .withHits(shiftedHits) - .withTargets(arrayOf(getFeature(CDR3)!!)) - } - - companion object { - const val COMMAND_NAME = MiXCRCommandDescriptor.reduceToCommonAssembleFeature.name - - private const val inputsLabel = "(input_file.(clns|clna)|directory)..." - - private const val outputLabel = "template.clns" - - - fun mkCommandSpec(): CommandSpec = - CommandSpec.forAnnotatedObject(CommandReduceToCommonAssembleFeature::class.java) - .addPositional( - CommandLine.Model.PositionalParamSpec.builder() - .index("0") - .required(false) - .arity("0..*") - .type(Path::class.java) - .paramLabel(inputsLabel) - .hideParamSyntax(true) - .description( - "Input files or directory with files for process.", - "In case of directory no filter by file type will be applied." - ) - .build() - ) - .addPositional( - CommandLine.Model.PositionalParamSpec.builder() - .index("1") - .required(false) - .arity("0..*") - .type(Path::class.java) - .paramLabel(outputLabel) - .hideParamSyntax(true) - .description(OutputTemplate.description) - .build() - ) - } -} diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt index 0791cdc07..b32b08424 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt @@ -221,10 +221,6 @@ object Main { ) .commandsGroup( CommandsGroup("Util commands") - .addSubcommand( - CommandReduceToCommonAssembleFeature.COMMAND_NAME, - CommandReduceToCommonAssembleFeature::class.java - ) .addSubcommand("exportReadsForClones", CommandExportReadsForClones::class.java) .addSubcommand("exportAlignmentsForClones", CommandExportAlignmentsForClones::class.java) .addSubcommand("exportReads", CommandExportReads::class.java) From 27ac3f8ee510c0d724aaaa9a60a87cdc6de934c3 Mon Sep 17 00:00:00 2001 From: gnefedev Date: Thu, 15 Feb 2024 12:38:41 +0100 Subject: [PATCH 3/6] update version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index cb4eb4449..44cb3be69 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -134,7 +134,7 @@ val toObfuscate: Configuration by configurations.creating { val obfuscationLibs: Configuration by configurations.creating -val mixcrAlgoVersion = "4.6.0-43-fixes" +val mixcrAlgoVersion = "4.6.0-46-reduceToCommonAssembleFeature" // may be blank (will be inherited from mixcr-algo) val milibVersion = "" // may be blank (will be inherited from mixcr-algo or milib) From e1598309fb70c2772c1ea59d79f55316e4c6908f Mon Sep 17 00:00:00 2001 From: gnefedev Date: Thu, 15 Feb 2024 12:53:11 +0100 Subject: [PATCH 4/6] fix tests --- .../com/milaboratory/mixcr/cli/CommandExportReports.kt | 4 ++-- .../mixcr/vdjaligners/VDJCAlignerParametersTest.java | 4 ++-- .../com/milaboratory/mixcr/cli/CommandFindAllelesTest.kt | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReports.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReports.kt index 75e8d83db..5d6594311 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReports.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReports.kt @@ -18,7 +18,7 @@ import com.milaboratory.app.InputFileType.YAML import com.milaboratory.app.ValidationException import com.milaboratory.app.logger import com.milaboratory.mixcr.basictypes.IOUtil -import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor +import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.getReportSafe import com.milaboratory.util.K_OM import com.milaboratory.util.K_YAML_OM @@ -130,7 +130,7 @@ class CommandExportReports : MiXCRCommandWithOutputs() { else -> footer.reports.collection.steps } steps.forEach { step -> - val command = AnalyzeCommandDescriptor.fromString(step) + val command = MiXCRCommandDescriptor.fromString(step) val reports = footer.reports.getReportSafe(command) if (reports == null) { helper.println("Can't read report for $step, file has too old version") diff --git a/src/test/java/com/milaboratory/mixcr/vdjaligners/VDJCAlignerParametersTest.java b/src/test/java/com/milaboratory/mixcr/vdjaligners/VDJCAlignerParametersTest.java index b862bf636..be8121814 100644 --- a/src/test/java/com/milaboratory/mixcr/vdjaligners/VDJCAlignerParametersTest.java +++ b/src/test/java/com/milaboratory/mixcr/vdjaligners/VDJCAlignerParametersTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2022, MiLaboratories Inc. All Rights Reserved + * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved * * Before downloading or accessing the software, please read carefully the * License Agreement available at: @@ -53,4 +53,4 @@ public void test1() throws Exception { VDJCAlignerParameters clone = deser.clone(); assertEquals(paramentrs, clone); } -} \ No newline at end of file +} diff --git a/src/test/kotlin/com/milaboratory/mixcr/cli/CommandFindAllelesTest.kt b/src/test/kotlin/com/milaboratory/mixcr/cli/CommandFindAllelesTest.kt index 9cc85f4d4..5a893af6b 100644 --- a/src/test/kotlin/com/milaboratory/mixcr/cli/CommandFindAllelesTest.kt +++ b/src/test/kotlin/com/milaboratory/mixcr/cli/CommandFindAllelesTest.kt @@ -15,11 +15,11 @@ import com.milaboratory.app.ValidationException import com.milaboratory.util.TempFileDest import com.milaboratory.util.TempFileManager import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder +import io.kotest.matchers.string.shouldStartWith import org.junit.Assert import org.junit.Before import org.junit.Test import picocli.CommandLine -import java.io.IOException import java.nio.file.Path import java.nio.file.Paths @@ -30,7 +30,6 @@ class CommandFindAllelesTest { private val file3: Path = temp.resolvePath("folder3/file1.clns") @Before - @Throws(IOException::class) fun prepareFiles() { file1.toFile().parentFile.mkdirs() file1.toFile().createNewFile() @@ -122,7 +121,7 @@ class CommandFindAllelesTest { command.outputFiles Assert.fail() } catch (e: ValidationException) { - Assert.assertTrue(e.message, e.message.startsWith("Output clns files are not uniq:")) + e.message shouldStartWith "Output files are not uniq:" } } @@ -136,7 +135,7 @@ class CommandFindAllelesTest { ).parseResult val command = p.asCommandLineList()[p.asCommandLineList().size - 1].getCommand() try { - command.outputFiles + command.validate() Assert.fail() } catch (e: ValidationException) { Assert.assertEquals( From 4283308357bac58ee1454232d2c8c1ad82b48d2b Mon Sep 17 00:00:00 2001 From: gnefedev Date: Thu, 15 Feb 2024 12:03:19 +0000 Subject: [PATCH 5/6] regression tests automated change --- regression/cli-help/findAlleles.txt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/regression/cli-help/findAlleles.txt b/regression/cli-help/findAlleles.txt index 8f5ede458..510265f3d 100644 --- a/regression/cli-help/findAlleles.txt +++ b/regression/cli-help/findAlleles.txt @@ -12,14 +12,13 @@ of V and J genes and the same features to align. In case of directory no filter by file type will be applied. --output-template Output template may contain {file_name} and {file_dir_path}, - outputs for '-o /output/folder/{file_name}_with_alleles.clns - input_file.clns input_file2.clns' will be - /output/folder/input_file_with_alleles.clns and - /output/folder/input_file2_with_alleles.clns, - outputs for '-o {file_dir_path}/{file_name}_with_alleles.clns + outputs for '-o /output/folder/{file_name}_suffix.clns input_file.clns + input_file2.clns' will be /output/folder/input_file_suffix.clns and + /output/folder/input_file2_suffix.clns, + outputs for '-o {file_dir_path}/{file_name}_suffix.clns /some/folder1/input_file.clns /some/folder2/input_file2.clns' will - be /seme/folder1/input_file_with_alleles.clns and - /some/folder2/input_file2_with_alleles.clns + be /seme/folder1/input_file_suffix.clns and + /some/folder2/input_file2_suffix.clns Resulted outputs must be uniq --no-clns-output Command will not realign input clns files. Must be specified if `--output-template` is omitted. @@ -81,7 +80,7 @@ of V and J genes and the same features to align. allele. --dont-remove-unused-genes Don't remove genes that weren't used in clones in the result library - -O Overrides default build SHM parameter values + -O Overrides default find alleles parameter values -r, --report Report file (human readable version, see `-j / --json-report` for machine readable report). -j, --json-report JSON formatted report file. From aa7cb9bd1ef2eb543127429007f1468ae9220d67 Mon Sep 17 00:00:00 2001 From: gnefedev Date: Fri, 16 Feb 2024 10:20:43 +0100 Subject: [PATCH 6/6] fix CommandExportReportsAsTable --- .../milaboratory/mixcr/cli/CommandExportReportsAsTable.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt index 01a23eef9..00527f9b0 100644 --- a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt +++ b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandExportReportsAsTable.kt @@ -23,7 +23,7 @@ import com.milaboratory.mixcr.export.MetaForExport import com.milaboratory.mixcr.export.ReportFieldsExtractors import com.milaboratory.mixcr.export.ReportFieldsExtractors.ReportsWithSource import com.milaboratory.mixcr.export.RowMetaForExport -import com.milaboratory.mixcr.presets.AnalyzeCommandDescriptor +import com.milaboratory.mixcr.presets.MiXCRCommandDescriptor import com.milaboratory.mixcr.presets.StepDataCollection import com.milaboratory.mixcr.presets.getReportSafe import picocli.CommandLine @@ -121,7 +121,7 @@ class CommandExportReportsAsTable : MiXCRCommandWithOutputs() { } upstreamRecords + ReportsWithSource( input.toString(), - AnalyzeCommandDescriptor.values().mapNotNull { footer.reports.getReportSafe(it) }.flatten(), + MiXCRCommandDescriptor.values().mapNotNull { footer.reports.getReportSafe(it) }.flatten(), upstreamRecords.flatMap { it.reports } ) } @@ -139,7 +139,7 @@ class CommandExportReportsAsTable : MiXCRCommandWithOutputs() { val upstreamReports = collection.upstreamReportsWithSources() upstreamReports + ReportsWithSource( sourceName, - AnalyzeCommandDescriptor.values().mapNotNull { collection.getReportSafe(it) }.flatten(), + MiXCRCommandDescriptor.values().mapNotNull { collection.getReportSafe(it) }.flatten(), upstreamReports.flatMap { it.reports } ) }