Merge pull request #1524 from milaboratory/fixes

fix fineFilteringPredicate
milaboratory · Feb 7, 2024 · e87cffb · e87cffb
2 parents 4b0613e + 27f9d0e
commit e87cffb
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 63 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -134,7 +134,7 @@ val toObfuscate: Configuration by configurations.creating {
 val obfuscationLibs: Configuration by configurations.creating
 
 
-val mixcrAlgoVersion = "4.6.0-36-fixes"
+val mixcrAlgoVersion = "4.6.0-38-fixes"
 // may be blank (will be inherited from mixcr-algo)
 val milibVersion = ""
 // may be blank (will be inherited from mixcr-algo or milib)

diff --git a/regression/reports/baseBuldTrees.yaml b/regression/reports/baseBuldTrees.yaml
@@ -973,6 +973,8 @@ upstreams:
             nonFunctional: 37
             isOOF: 32
             hasStops: 5
+      clonesFilteredInFineFiltering: 0
+      readsFilteredInFineFiltering: 0.0
       clonesFilteredInPostFiltering: 0
       readsFilteredInPostFiltering: 0.0
       postFilteringReports: null
@@ -1883,6 +1885,8 @@ upstreams:
             nonFunctional: 1
             isOOF: 1
             hasStops: 0
+      clonesFilteredInFineFiltering: 0
+      readsFilteredInFineFiltering: 0.0
       clonesFilteredInPostFiltering: 0
       readsFilteredInPostFiltering: 0.0
       postFilteringReports: null

diff --git a/regression/reports/baseSingleCell.raw.yaml b/regression/reports/baseSingleCell.raw.yaml
@@ -206,6 +206,8 @@ assemble:
         nonFunctional: 0
         isOOF: 0
         hasStops: 0
+  clonesFilteredInFineFiltering: 0
+  readsFilteredInFineFiltering: 0.0
   clonesFilteredInPostFiltering: 0
   readsFilteredInPostFiltering: 0.0
   postFilteringReports:

diff --git a/regression/reports/baseSingleCell.vdjcontigs.yaml b/regression/reports/baseSingleCell.vdjcontigs.yaml
@@ -207,6 +207,8 @@ assemble:
         nonFunctional: 0
         isOOF: 0
         hasStops: 0
+  clonesFilteredInFineFiltering: 0
+  readsFilteredInFineFiltering: 0.0
   clonesFilteredInPostFiltering: 0
   readsFilteredInPostFiltering: 0.0
   postFilteringReports:

diff --git a/regression/schemas/reports/CloneAssemblerReport.schema.yaml b/regression/schemas/reports/CloneAssemblerReport.schema.yaml
@@ -168,6 +168,8 @@ properties:
     type: integer
   clonesDroppedAsLowQuality:
     type: integer
+  clonesFilteredInFineFiltering:
+    type: integer
   clonesFilteredInPostFiltering:
     type: integer
   clonesPreClustered:
@@ -268,6 +270,8 @@ properties:
     type: integer
   readsDroppedWithLowQualityClones:
     type: integer
+  readsFilteredInFineFiltering:
+    type: number
   readsFilteredInPostFiltering:
     type: number
   readsInClones:

diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/CommandAssembleContigs.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved
+ * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved
  *
  * Before downloading or accessing the software, please read carefully the
  * License Agreement available at:
@@ -13,7 +13,6 @@
 
 package com.milaboratory.mixcr.cli
 
-import cc.redberry.pipe.CUtils
 import cc.redberry.pipe.util.forEach
 import cc.redberry.pipe.util.mapInParallelOrdered
 import com.milaboratory.app.InputFileType
@@ -46,14 +45,12 @@ import com.milaboratory.primitivio.PrimitivI
 import com.milaboratory.primitivio.PrimitivO
 import com.milaboratory.util.ReportUtil
 import com.milaboratory.util.SmartProgressReporter
-import com.milaboratory.util.StreamUtil
 import io.repseq.core.GeneFeature
 import io.repseq.core.GeneFeatures
 import io.repseq.core.GeneType
 import io.repseq.core.GeneType.Joining
 import io.repseq.core.GeneType.Variable
 import io.repseq.core.VDJCGene
-import io.repseq.core.VDJCGeneId
 import io.repseq.core.VDJCLibraryRegistry
 import picocli.CommandLine.Command
 import picocli.CommandLine.Mixin
@@ -67,7 +64,6 @@ import java.io.FileOutputStream
 import java.io.OutputStreamWriter
 import java.nio.file.Path
 import java.util.*
-import java.util.stream.Collectors
 
 object CommandAssembleContigs {
     const val COMMAND_NAME = MiXCRCommandDescriptor.assembleContigs.name
@@ -221,75 +217,37 @@ object CommandAssembleContigs {
                                 try {
                                     // Collecting statistics
                                     var coverages = clone.hitsMap
-                                        .entries.stream()
-                                        .filter { (_, value) -> value != null && value.isNotEmpty() }
-                                        .collect(
-                                            Collectors.toMap(
-                                                { (key, _) -> key },
-                                                { (_, value) ->
-                                                    Arrays.stream(
-                                                        value
-                                                    )
-                                                        .filter { h ->
-                                                            h.geneType != Variable && h.geneType != Joining ||
-                                                                    FullSeqAssembler.checkGeneCompatibility(
-                                                                        h,
-                                                                        reader.assemblingFeatures
-                                                                    )
-                                                        }
-                                                        .collect(
-                                                            Collectors.toMap(
-                                                                { h -> h.gene.id },
-                                                                { hit -> CoverageAccumulator(hit) })
+                                        .filterValues { value -> value != null && value.isNotEmpty() }
+                                        .mapValuesTo(EnumMap(GeneType::class.java)) { (_, value) ->
+                                            value.filter { hit ->
+                                                hit.geneType !in GeneType.VJ_REFERENCE ||
+                                                        FullSeqAssembler.checkGeneCompatibility(
+                                                            hit, reader.assemblingFeatures
                                                         )
-                                                },
-                                                StreamUtil.noMerge(),
-                                                { EnumMap(GeneType::class.java) }
-                                            )
-                                        )
+                                            }.associate { hit -> hit.gene.id to CoverageAccumulator(hit) }
+                                        }
 
                                     // Filtering empty maps
-                                    coverages = coverages.entries.stream()
-                                        .filter { (_, value) -> value.isNotEmpty() }
-                                        .collect(
-                                            Collectors.toMap(
-                                                { (key, _) -> key },
-                                                { (_, value) -> value },
-                                                StreamUtil.noMerge(),
-                                                { EnumMap(GeneType::class.java) }
-                                            )
-                                        )
-                                    if (!coverages.containsKey(Variable) || !coverages.containsKey(Joining)) {
+                                    coverages = coverages
+                                        .filterValues { it.isNotEmpty() }
+                                        .toMap(EnumMap(GeneType::class.java))
+                                    if (Variable !in coverages || Joining !in coverages) {
                                         // Something went really wrong
                                         reportBuilder.onAssemblyCanceled(clone)
                                         return@mapInParallelOrdered arrayOf(clone)
                                     }
-                                    for (alignments in CUtils.it(cloneAlignments.alignments())) {
+                                    cloneAlignments.alignments().forEach { alignments ->
                                         for ((key, value) in alignments.hitsMap) {
                                             for (hit in value) {
-                                                Optional.ofNullable(coverages[key])
-                                                    .flatMap { Optional.ofNullable(it[hit.gene.id]) }
-                                                    .ifPresent { acc -> acc.accumulate(hit) }
+                                                coverages[key]?.let { it[hit.gene.id]?.accumulate(hit) }
                                             }
                                         }
                                     }
 
                                     // Selecting best hits for clonal sequence assembly based in the coverage information
-                                    val bestGenes = coverages.entries.stream()
-                                        .collect(
-                                            Collectors.toMap(
-                                                { (key, _) -> key },
-                                                { (_, value) ->
-                                                    value.entries.stream()
-                                                        .max(Comparator.comparing { (_, value1): Map.Entry<VDJCGeneId?, CoverageAccumulator> ->
-                                                            value1.getNumberOfCoveredPoints(1)
-                                                        })
-                                                        .map { (_, value1) -> value1.hit }
-                                                        .get()
-                                                },
-                                                StreamUtil.noMerge(),
-                                                { EnumMap(GeneType::class.java) })
-                                        )
+                                    val bestGenes = coverages.mapValuesTo(EnumMap(GeneType::class.java)) { (_, value) ->
+                                        value.values.maxByOrNull { it.getNumberOfCoveredPoints(1) }?.hit
+                                    }
 
                                     // Performing contig assembly
                                     val fullSeqAssembler = FullSeqAssembler(
@@ -302,7 +260,6 @@ object CommandAssembleContigs {
                                     val rawVariantsData =
                                         fullSeqAssembler.calculateRawData { cloneAlignments.alignments() }
                                     if (debugReport != null) {
-                                        @Suppress("BlockingMethodInNonBlockingContext")
                                         synchronized(debugReport) {
                                             FileOutputStream(debugReportFile?.toString() + "." + clone.id).use { fos ->
                                                 val content = rawVariantsData.toCsv(10.toByte())

diff --git a/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt b/src/main/kotlin/com/milaboratory/mixcr/cli/Main.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2023, MiLaboratories Inc. All Rights Reserved
+ * Copyright (c) 2014-2024, MiLaboratories Inc. All Rights Reserved
  *
  * Before downloading or accessing the software, please read carefully the
  * License Agreement available at:
@@ -68,6 +68,7 @@ import java.lang.management.ManagementFactory
 import java.nio.file.Files
 import java.nio.file.Paths
 import kotlin.io.path.Path
+import kotlin.io.path.absolute
 import kotlin.system.exitProcess
 
 object Main {
@@ -385,6 +386,12 @@ object Main {
                     err.println("   Version: " + MiXCRVersionInfo.get().shortestVersionString)
                     err.println("        OS: " + System.getProperty("os.name"))
                     err.println("      Java: " + System.getProperty("java.version"))
+                    err.println("  Abs path: " + Path("").absolute())
+                    if (logger.verbose)
+                        err.println(" Libraries: " + VDJCLibraryRegistry.getDefault().loadedLibraries.map { library ->
+                            val sources = library.data.genes.map { it.baseSequence.origin }.distinct()
+                            "`${library.name}:${library.taxonId}` with sources $sources"
+                        })
                     err.println("  Cmd args: " + cmdArgs.joinToString(" "))
                     try {
                         ex.rethrowOOM()