Skip to content

Commit

Permalink
Merge branch 'release/v3.0.9'
Browse files Browse the repository at this point in the history
  • Loading branch information
dbolotin committed Aug 6, 2019
2 parents 98f1240 + 80a6afd commit 1bc4875
Show file tree
Hide file tree
Showing 21 changed files with 136 additions and 20 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@

MiXCR 3.0.9 ( 6 Aug 2019)
========================

-- Chain usage statistics added to `align` and `assemble` JSON reports
-- Fixed wrong behaviour with score-based pre-filtering in `split-by-V/J=true` cases
-- Fixed rare IndexOutOfBounds exception in `-nFeatureImputed ...`
-- Added shortcut for `--json-report` = `-j`
-- Sanity check for common mistake in `analyze` parameters


MiXCR 3.0.8 (22 Jul 2019)
========================

Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

<groupId>com.milaboratory</groupId>
<artifactId>mixcr</artifactId>
<version>3.0.8</version>
<version>3.0.9</version>
<packaging>jar</packaging>
<name>MiXCR</name>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,7 @@ public List<CloneAccumulator> build() {
final TIntObjectHashMap<TIntArrayList> reversePreClustered = new TIntObjectHashMap<>();

Arrays.sort(accs, CLONE_ACCUMULATOR_COMPARATOR);
double minCountForMaxScore = accs[0].getCount() / parameters.preClusteringCountFilteringRatio;
int deleted = 0;

for (int i = 0; i < accs.length - 1; i++) {
Expand Down Expand Up @@ -677,10 +678,13 @@ public List<CloneAccumulator> build() {
// Score filtering step

// Calculation

float[] maxScores = new float[2];
for (CloneAccumulator acc : accs) {
if (acc == null)
continue;
if (acc.getCount() < minCountForMaxScore)
continue;
for (int i = 0; i < 2; i++) { // Only for V and J
GeneType gt = GeneType.VJC_REFERENCE[i];
maxScores[i] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public final class CloneAssemblerParameters implements java.io.Serializable {
boolean separateByV, separateByJ, separateByC;
double maximalPreClusteringRatio;
double preClusteringScoreFilteringRatio;
double preClusteringCountFilteringRatio;
boolean addReadsCountOnClustering;
byte badQualityThreshold;
double maxBadPointsPercent;
Expand All @@ -75,6 +76,7 @@ public CloneAssemblerParameters(@JsonProperty("assemblingFeatures") GeneFeature[
@JsonProperty("separateByC") boolean separateByC,
@JsonProperty("maximalPreClusteringRatio") double maximalPreClusteringRatio,
@JsonProperty("preClusteringScoreFilteringRatio") double preClusteringScoreFilteringRatio,
@JsonProperty("preClusteringCountFilteringRatio") double preClusteringCountFilteringRatio,
@JsonProperty("addReadsCountOnClustering") boolean addReadsCountOnClustering,
@JsonProperty("badQualityThreshold") byte badQualityThreshold,
@JsonProperty("maxBadPointsPercent") double maxBadPointsPercent,
Expand All @@ -90,6 +92,7 @@ public CloneAssemblerParameters(@JsonProperty("assemblingFeatures") GeneFeature[
this.separateByC = separateByC;
this.maximalPreClusteringRatio = maximalPreClusteringRatio;
this.preClusteringScoreFilteringRatio = preClusteringScoreFilteringRatio;
this.preClusteringCountFilteringRatio = preClusteringCountFilteringRatio;
this.addReadsCountOnClustering = addReadsCountOnClustering;
this.badQualityThreshold = badQualityThreshold;
this.maxBadPointsPercent = maxBadPointsPercent;
Expand Down Expand Up @@ -264,6 +267,11 @@ public CloneAssemblerParameters setPreClusteringScoreFilteringRatio(double preCl
return this;
}

public CloneAssemblerParameters setPreClusteringCountFilteringRatio(double preClusteringCountFilteringRatio) {
this.preClusteringCountFilteringRatio = preClusteringCountFilteringRatio;
return this;
}


public CloneAssemblerParameters setAddReadsCountOnClustering(boolean addReadsCountOnClustering) {
this.addReadsCountOnClustering = addReadsCountOnClustering;
Expand All @@ -289,7 +297,7 @@ public CloneAssemblerParameters clone() {
qualityAggregationType,
cloneClusteringParameters == null ? null : cloneClusteringParameters.clone(),
cloneFactoryParameters.clone(), separateByV, separateByJ, separateByC,
maximalPreClusteringRatio, preClusteringScoreFilteringRatio, addReadsCountOnClustering, badQualityThreshold, maxBadPointsPercent,
maximalPreClusteringRatio, preClusteringScoreFilteringRatio, preClusteringCountFilteringRatio, addReadsCountOnClustering, badQualityThreshold, maxBadPointsPercent,
mappingThreshold, minimalQuality);
}

Expand All @@ -307,6 +315,7 @@ public boolean equals(Object o) {
if (separateByC != that.separateByC) return false;
if (Double.compare(that.maximalPreClusteringRatio, maximalPreClusteringRatio) != 0) return false;
if (Double.compare(that.preClusteringScoreFilteringRatio, preClusteringScoreFilteringRatio) != 0) return false;
if (Double.compare(that.preClusteringCountFilteringRatio, preClusteringCountFilteringRatio) != 0) return false;
if (addReadsCountOnClustering != that.addReadsCountOnClustering) return false;
if (badQualityThreshold != that.badQualityThreshold) return false;
if (Double.compare(that.maxBadPointsPercent, maxBadPointsPercent) != 0) return false;
Expand Down Expand Up @@ -338,6 +347,8 @@ public int hashCode() {
result = 31 * result + (int) (temp ^ (temp >>> 32));
temp = Double.doubleToLongBits(preClusteringScoreFilteringRatio);
result = 31 * result + (int) (temp ^ (temp >>> 32));
temp = Double.doubleToLongBits(preClusteringCountFilteringRatio);
result = 31 * result + (int) (temp ^ (temp >>> 32));
result = 31 * result + (addReadsCountOnClustering ? 1 : 0);
result = 31 * result + (int) badQualityThreshold;
temp = Double.doubleToLongBits(maxBadPointsPercent);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,11 @@ private void clusterizeBranches(int[] points, List<VariantBranch> branches) {
sumWeight += weights[j - i - 1];
}

for (int j = i + 1; j < branches.size(); ++j) {
VariantBranch cluster = branches.get(j);
cluster.count += branch.count * weights[j - i - 1] / sumWeight;
}
if (sumWeight != 0.0)
for (int j = i + 1; j < branches.size(); ++j) {
VariantBranch cluster = branches.get(j);
cluster.count += branch.count * weights[j - i - 1] / sumWeight;
}

if (report != null)
report.onVariantClustered(branch);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,9 @@ public CaseSensitiveNucleotideSequence getIncompleteFeature(GeneFeature geneFeat
if (lLast.begin > rLast.end)
return null;

if (lLast.germline || rLast.germline)
return null;

// assert lHit.getGene().getGeneType() == GeneType.Variable;
// if (!lHit
// .getPartitioningForTarget(lLast.iTarget)
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/com/milaboratory/mixcr/cli/AlignerReport.java
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,11 @@ public long getJChimeras() {
return jChimeras.get();
}

@JsonProperty("chainUsage")
public ChainUsageStats getChainUsage() {
return chainStats;
}

@JsonProperty("realignedWithForcedNonFloatingBound")
public long getRealignedWithForcedNonFloatingBound() {
return realignedWithForcedNonFloatingBound.get();
Expand Down
27 changes: 26 additions & 1 deletion src/main/java/com/milaboratory/mixcr/cli/ChainUsageStats.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,24 @@
*/
package com.milaboratory.mixcr.cli;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.milaboratory.mixcr.basictypes.VDJCObject;
import io.repseq.core.Chains;

import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;

/**
* Created by poslavsky on 08/11/2016.
*/
final class ChainUsageStats implements Report {
@JsonSerialize(using = ChainUsageStats.Serializer.class)
public final class ChainUsageStats implements Report {
final AtomicLong chimeras = new AtomicLong(0);
final AtomicLong total = new AtomicLong(0);
final ConcurrentHashMap<Chains, AtomicLong> counters = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -80,4 +87,22 @@ public void writeReport(ReportHelper helper) {
for (Map.Entry<Chains, AtomicLong> ch : counters.entrySet())
helper.writePercentAndAbsoluteField(ch.getKey().toString() + " chains", ch.getValue(), total);
}

public static final class Serializer extends JsonSerializer<ChainUsageStats> {
@Override
public void serialize(ChainUsageStats value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException {
jgen.writeStartObject();
jgen.writeNumberField("total", value.total.longValue());
jgen.writeNumberField("chimeras", value.chimeras.longValue());
jgen.writeObjectFieldStart("chains");
for (Map.Entry<Chains, AtomicLong> entry : value.counters.entrySet()) {
String chains = entry.getKey().toString();
if (chains.isEmpty())
chains = "X";
jgen.writeNumberField(chains, entry.getValue().longValue());
}
jgen.writeEndObject();
jgen.writeEndObject();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,11 @@ public long getReadsDroppedWithClones() {
return readsDroppedWithClones.get();
}

@JsonProperty("clonalChainUsage")
public ChainUsageStats getClonalChainUsage() {
return chainStats;
}

@Override
public void onNewCloneCreated(CloneAccumulator accumulator) {
clonesCreated.incrementAndGet();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ protected List<String> getOutputFiles() {
public String reportFile = null;

@Option(description = CommonDescriptions.JSON_REPORT,
names = {"--json-report"})
names = {"-j", "--json-report"})
public String jsonReport = null;

@Option(description = "V/D/J/C gene library",
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;

import java.io.File;
import java.lang.reflect.Field;
import java.util.*;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -623,6 +624,9 @@ public void validate() {
super.validate();
if (report == null)
warn("NOTE: report file is not specified, using " + getReport() + " to write report.");
if (new File(outputNamePattern()).exists())
throwValidationException("Output file name prefix, matches the existing file name. Most probably you " +
"specified paired-end file names but forgot to specify output file name prefix.", false);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void setThreads(int threads) {
public String reportFile;

@Option(description = CommonDescriptions.JSON_REPORT,
names = {"--json-report"})
names = {"-j", "--json-report"})
public String jsonReport;

@Option(description = "Show buffer statistics.",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void setThreads(int threads) {
public String debugReportFile;

@Option(description = CommonDescriptions.JSON_REPORT,
names = {"--json-report"})
names = {"-j", "--json-report"})
public String jsonReport = null;

public FullSeqAssemblerParameters getFullSeqAssemblerParameters() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public class CommandAssemblePartialAlignments extends ACommandWithSmartOverwrite
public String reportFile;

@Option(description = CommonDescriptions.JSON_REPORT,
names = {"--json-report"})
names = {"-j", "--json-report"})
public String jsonReport = null;

@Option(description = "Write only overlapped sequences (needed for testing).",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public class CommandExtend extends ACommandWithSmartOverwriteWithSingleInputMiXC
public String reportFile;

@Option(description = CommonDescriptions.JSON_REPORT,
names = {"--json-report"})
names = {"-j", "--json-report"})
public String jsonReport = null;

@Option(description = "Quality score value to assign imputed sequences",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ private CommonDescriptions() {
"Species (organism), as specified in library file or taxon id.%n" +
"Possible values: hs, HomoSapiens, musmusculus, mmu, hsa, 9606, 10090 etc.";

public static final String REPORT = "Report file (human readable version, see --json-report for machine readable report)";
public static final String REPORT = "Report file (human readable version, see -j / --json-report for machine readable report)";

public static final String JSON_REPORT = "JSON formatted report file";

Expand Down
1 change: 1 addition & 0 deletions src/main/resources/parameters/assembler_parameters.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"separateByC": false,
"maximalPreClusteringRatio": 1.0,
"preClusteringScoreFilteringRatio": 2.0,
"preClusteringCountFilteringRatio": 2.0,
"cloneClusteringParameters": {
"searchDepth": 2,
"allowedMutationsInNRegions": 1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ null, new DClonalAlignerParameters(0.85f, 30.0f, 3, AffineGapAlignmentScoring.ge
CloneAssemblerParameters params = new CloneAssemblerParameters(new GeneFeature[]{GeneFeature.FR1, GeneFeature.CDR3}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4, 2.0, true, (byte) 20, .8, "2", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2", (byte) 15);

String str = GlobalObjectMappers.PRETTY.writeValueAsString(params);
//System.out.println(str);
Expand Down Expand Up @@ -82,7 +82,7 @@ null, new DClonalAlignerParameters(0.85f, 30.0f, 3, AffineGapAlignmentScoring.ge
CloneAssemblerParameters params = new CloneAssemblerParameters(new GeneFeature[]{GeneFeature.FR1, GeneFeature.CDR3}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4, 2.0, true, (byte) 20, .8, "2of6", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2of6", (byte) 15);

String str = GlobalObjectMappers.PRETTY.writeValueAsString(params);
//System.out.println(str);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ private static CloneSet runFullPipeline(String... fastqFiles) throws IOException
new GeneFeature[]{GeneFeature.CDR3}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4,2.0, true, (byte) 20, .8, "2 of 6", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2 of 6", (byte) 15);

System.out.println(GlobalObjectMappers.toOneLine(assemblerParameters));

Expand All @@ -130,7 +130,7 @@ private static CloneSet runFullPipeline(String... fastqFiles) throws IOException
CloneSet cloneSet = assemblerRunner.getCloneSet(null);

ByteArrayOutputStream bos = new ByteArrayOutputStream();
try(ClnsWriter writer = new ClnsWriter(null, cloneSet, bos)){
try (ClnsWriter writer = new ClnsWriter(null, cloneSet, bos)) {
writer.write();
}

Expand Down
47 changes: 47 additions & 0 deletions src/test/java/com/milaboratory/mixcr/cli/ChainUsageStatsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
* (here and after addressed as Inventors)
* All Rights Reserved
*
* Permission to use, copy, modify and distribute any part of this program for
* educational, research and non-profit purposes, by non-profit institutions
* only, without fee, and without a written agreement is hereby granted,
* provided that the above copyright notice, this paragraph and the following
* three paragraphs appear in all copies.
*
* Those desiring to incorporate this work into commercial products or use for
* commercial purposes should contact MiLaboratory LLC, which owns exclusive
* rights for distribution of this program for commercial purposes, using the
* following email address: licensing@milaboratory.com.
*
* IN NO EVENT SHALL THE INVENTORS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
* SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
* ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE INVENTORS HAS BEEN
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* THE SOFTWARE PROVIDED HEREIN IS ON AN "AS IS" BASIS, AND THE INVENTORS HAS
* NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
* MODIFICATIONS. THE INVENTORS MAKES NO REPRESENTATIONS AND EXTENDS NO
* WARRANTIES OF ANY KIND, EITHER IMPLIED OR EXPRESS, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
* PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY
* PATENT, TRADEMARK OR OTHER RIGHTS.
*/
package com.milaboratory.mixcr.cli;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.milaboratory.util.GlobalObjectMappers;
import io.repseq.core.Chains;
import org.junit.Test;

public class ChainUsageStatsTest {
@Test
public void serializationTest() throws JsonProcessingException {
ChainUsageStats stats = new ChainUsageStats();
stats.total.incrementAndGet();
stats.total.incrementAndGet();
stats.chimeras.incrementAndGet();
stats.getCounter(Chains.TRB).incrementAndGet();
System.out.println(GlobalObjectMappers.PRETTY.writeValueAsString(stats));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ null, new DClonalAlignerParameters(0.85f, 30.0f, 3, AffineGapAlignmentScoring.ge
CloneAssemblerParameters params = new CloneAssemblerParameters(new GeneFeature[]{GeneFeature.FR1, GeneFeature.CDR3}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4,2.0, true, (byte) 20, .8, "2of6", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2of6", (byte) 15);

CloneAssemblerParameters override = JsonOverrider.override(
params,
Expand All @@ -108,7 +108,7 @@ null, new DClonalAlignerParameters(0.85f, 30.0f, 3, AffineGapAlignmentScoring.ge
CloneAssemblerParameters expected = new CloneAssemblerParameters(new GeneFeature[]{new GeneFeature(GeneFeature.CDR1, -5, +6), GeneFeature.CDR2}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4,2.0, true, (byte) 20, .8, "2of6", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2of6", (byte) 15);


Assert.assertEquals(expected, override);
Expand All @@ -127,7 +127,7 @@ null, new DClonalAlignerParameters(0.85f, 30.0f, 3, AffineGapAlignmentScoring.ge
CloneAssemblerParameters params = new CloneAssemblerParameters(new GeneFeature[]{GeneFeature.FR1, GeneFeature.CDR3}, 12,
QualityAggregationType.Average,
new CloneClusteringParameters(2, 1, TreeSearchParameters.ONE_MISMATCH, new RelativeConcentrationFilter(1.0E-6)),
factoryParameters, true, true, false, 0.4,2.0, true, (byte) 20, .8, "2of6", (byte) 15);
factoryParameters, true, true, false, 0.4, 2.0, 2.0, true, (byte) 20, .8, "2of6", (byte) 15);

CloneAssemblerParameters override = JsonOverrider.override(
params,
Expand Down

0 comments on commit 1bc4875

Please sign in to comment.