Merge branch 'release/2.0.0'

.dockerignore

@@ -0,0 +1,6 @@
+/cwls/
+/examples/
+/images/
+/git-hooks/
+/*.*
+/LICENSE

.travis.yml

@@ -8,7 +8,40 @@ services:
   - docker
 
 install:
- - docker build -t dockstore-cgpwgs .
+ - virtualenv -p python3 venv
+ - source venv/bin/activate
+ - pip install html5lib cwltool
 
 script:
+ - set -e
+ - echo 'Validate CWL file(s)'
+ - cwltool --validate cwls/cgpwgs.cwl
+ - echo 'Build and check docker image'
+ - docker build -t dockstore-cgpwgs .
  - docker images | grep -c dockstore-cgpwgs
+ - echo 'Verify program from each inherited package is found (dockstore-cgpbigwig)'
+ - docker run -t --rm dockstore-cgpwgs bwjoin --version
+ - echo 'Verify program from each inherited package is found (dockstore-cgpmap)'
+ - docker run -t --rm dockstore-cgpwgs ds-cgpmap.pl -h
+ - docker run -t --rm dockstore-cgpwgs bwa_mem.pl -version
+ - docker run -t --rm dockstore-cgpwgs bammarkduplicates2 --version
+ - docker run -t --rm dockstore-cgpwgs samtools --version
+ - docker run -t --rm dockstore-cgpwgs bash -c 'bwa 2>&1 | grep Version'
+ - echo 'Verify program from each inherited package is found (dockstore-cgpwxs)'
+ - docker run -t --rm dockstore-cgpwgs ds-cgpwxs.pl -h
+ - docker run -t --rm dockstore-cgpwgs cgpVCFSplit.pl -version
+ - docker run -t --rm dockstore-cgpwgs vcftools --version
+ - docker run -t --rm dockstore-cgpwgs bedtools --version
+ - docker run -t --rm dockstore-cgpwgs bash -c 'pindel 2>&1 | grep Welcome'
+ - docker run -t --rm dockstore-cgpwgs pindel.pl -version
+ - docker run -t --rm dockstore-cgpwgs caveman --version
+ - docker run -t --rm dockstore-cgpwgs caveman.pl -version
+ - docker run -t --rm dockstore-cgpwgs cgpFlagCaVEMan.pl -version
+ - docker run -t --rm dockstore-cgpwgs AnnotateVcf.pl -version
+ - echo 'Verify a program from each new package is found (dockstore-cgpwgs)'
+ - docker run -t --rm dockstore-cgpwgs ds-cgpwgs.pl -h
+ - docker run -t --rm dockstore-cgpwgs bash -c 'verifyBamID 2>&1 | grep verifyBamID'
+ - docker run -t --rm dockstore-cgpwgs alleleCounter --version
+ - docker run -t --rm dockstore-cgpwgs brass.pl -version
+ - docker run -t --rm dockstore-cgpwgs ascat.pl -version
+ - docker run -t --rm dockstore-cgpwgs battenberg.pl -version

CHANGES.md

@@ -1,21 +1,40 @@
-### 1.1.2
+# CHANGES
+
+## 2.0.0
+
+* Full support for BAM+CSI and CRAM+CRAI as inputs
+* Reorganisation to allow access of other tools or necessary bindings
+* See dockstore-cgpmap 3.0.0 - 3.0.4 (primarily adds mismatchQc and threads to some tools)
+* See dockstore-cgpwxs 3.0.0 - 3.0.3 (fragment based analysis)
+  * You will need to update the reference pack to include the new flagging rules, see example `json`
+  files
+* Using build stages to shrink images.
+* remove legacy PRE/POST-EXEC from cgpbox days
+* resolves #17, #18, #19, #20, #21, #24
+
+## 1.1.2
+
 * BRASS, random seed fixed to make noisy data have a reproducible result:
   * see [v6.0.4](https://github.com/cancerit/BRASS/releases/tag/v6.0.4)
 
-### 1.1.1
+## 1.1.1
+
 * Stable results between replicate runs:
   * see [v6.0.3](https://github.com/cancerit/BRASS/releases/tag/v6.0.3)
   * took multiple executions of 29 datasets to find this edge-case.
 
-### 1.1.0
+## 1.1.0
+
 BRASS updates
+
 * Significantly reduces the run time of brass_group phase
   * see [v6.0.0](https://github.com/cancerit/BRASS/releases/tag/v6.0.0)
   * Requires new version of the CNV_SV reference archive
 * Ensure stable result between replicate runs:
   * see [v6.0.2](https://github.com/cancerit/BRASS/releases/tag/v6.0.2)
 
 Other dependency updates
+
 * Based on [dockstore-cgpwxs:2.1.1](https://github.com/cancerit/dockstore-cgpwxs/releases/tag/2.1.1)
   * Update caveman, pindel and vagrent to improve reproducibility.
   * Reduced I/O in caveman.
@@ -24,50 +43,60 @@ Other dependency updates
   * [cgpBattenberg:v3.1.0](https://github.com/cancerit/cgpBattenberg/releases/tag/v3.1.0)
 
 Workflow updates:
+
 * Reorganisation of some processes to reduce run-time:
   * Changes to alleleCount have vastly improved I/O pattern and reduced runtime for
 ascat and IMPUTE-allele counts (pre-Battenberg).
   * Reduced number of split files for IMPUTE-allele counts as reduced run time negates need for large spread
   * Reduced alleleCount runtim allowed reorganisation of processes reducing runtime on example data from 23h to 19.5h (28cpu).
 * Added `cavereads` to example json specifying a higher value than default to reduce jobs required.
 
-### 1.0.8
+## 1.0.8
+
 * Update [Brass to v5.4.1](https://github.com/cancerit/BRASS/releases/tag/v5.4.1)
   * Provide additional reference file rather than attempt to decode species/build
   due to the may ways a species can be defined.
 
-### 1.0.7
+## 1.0.7
+
 * Update [BRASS to v5.4.0](https://github.com/cancerit/BRASS/releases/tag/v5.4.0)
   * Specifically fixes a vector merge bug which produces warnings (but no apparent change to results).
 
-### 1.0.6
+## 1.0.6
+
 * Fixes #13 - Parsing of sample name from BAM headers fails is sample at end of line
 * Update dockstore-cgpwxs to [2.0.7](https://github.com/cancerit/dockstore-cgpwxs/releases/tag/2.0.7) - Fixes bug in pindel core.
 
-### 1.0.5
+## 1.0.5
+
 * Improved handling of CPU oversubscription via PCAP-core update in base image
 * Update cgpCaVEManWrapper to expose extra options (specifically split size)
 
-### 1.0.4
+## 1.0.4
+
 Fix an error in param capture test
 
-### 1.0.3
+## 1.0.3
+
 * Export the `PCAP_THREADED_*` variables
 * Fixed #8, #9, #10
 * Updated base image to pull in fix required for `PCAP_THREADED_LOADBACKOFF`
 
-### 1.0.2
+## 1.0.2
+
 * Update BRASS to handle data that is _very_ quiet
 * Fix up the wrapper timings file so it is captured properly
 * Upgraded base image to [dockstore-cgpwxs:2.0.4](https://github.com/cancerit/dockstore-cgpwxs/releases/tag/2.0.4)
   * To pick up changes in [cgpPindel:2.2.0](https://github.com/cancerit/cgpPindel/releases/tag/v2.2.0)
 * Remove ability to build `*.bas` files as not possible to have optional secondary files and we state expectation of [dockstore-cgpmap](https://github.com/cancerit/dockstore-cgpmap) as data source.
 * Moved some processing around to reduce cpu wastage.
 
-### 1.0.1
+## 1.0.1
+
 * Update dependencies to reduce reliance on Capture::Tiny, apparent cause of some issues
 
-### 1.0.0
+## 1.0.0
+
 * Test data in `examples/analysis_config.local.json` moved to a non-expiring location.
 * See [dockstore-cgpwxs:2.0.0](https://github.com/cancerit/dockstore-cgpwxs/releases/tag/2.0.0)
 * Significant shrinking of image.

Dockerfile

@@ -1,40 +1,115 @@
-FROM  quay.io/wtsicgp/dockstore-cgpwxs:2.1.1
+FROM  quay.io/wtsicgp/dockstore-cgpwxs:3.0.3 as builder
 
-MAINTAINER  keiranmraine@gmail.com
+USER  root
 
-LABEL uk.ac.sanger.cgp="Cancer Genome Project, Wellcome Trust Sanger Institute" \
-      version="1.1.2" \
-      description="The CGP WGS pipeline for dockstore.org"
+RUN bash -c 'apt-get update -yq >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'apt-get install -qy --no-install-recommends lsb-release >& this.log || (cat this.log 1>&2 && exit 1)'
 
-USER  root
+RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" >> /etc/apt/sources.list
+RUN gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
+RUN gpg -a --export E084DAB9 | apt-key add -
+
+RUN bash -c 'apt-get update -yq >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'apt-get install -qy --no-install-recommends\
+  locales\
+  libcurl4-openssl-dev\
+  libssl-dev\
+  libssh2-1-dev\
+  libxml2-dev\
+  g++\
+  make\
+  gcc\
+  pkg-config\
+  zlib1g-dev\
+  libreadline6-dev\
+  libcairo2-dev\
+  gfortran\
+  unzip\
+  libboost-all-dev\
+  libpstreams-dev\
+  r-base\
+  r-base-dev\
+  libblas-dev\
+  libbz2-dev\
+     >& this.log || (cat this.log 1>&2 && exit 1)'
+
+RUN bash -c 'locale-gen en_US.UTF-8 >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'update-locale LANG=en_US.UTF-8 >& this.log || (cat this.log 1>&2 && exit 1)'
 
 ENV OPT /opt/wtsi-cgp
-ENV PATH $OPT/bin:$PATH
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
 ENV PERL5LIB $OPT/lib/perl5
 ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
 ENV R_LIBS $OPT/R-lib
 ENV R_LIBS_USER $R_LIBS
 
-COPY build/apt-build.sh build/
-RUN bash build/apt-build.sh
-
 COPY build/rlib-build.R build/
-RUN mkdir -p $R_LIBS_USER && Rscript build/rlib-build.R $R_LIBS_USER
-
-COPY build/perllib-build.sh build/
-RUN bash build/perllib-build.sh
+RUN mkdir -p $R_LIBS_USER
+RUN Rscript build/rlib-build.R $R_LIBS_USER 2>&1 | grep '^\*'
 
 COPY build/opt-build.sh build/
 RUN bash build/opt-build.sh $OPT
 
+FROM  ubuntu:16.04
+
+MAINTAINER  keiranmraine@gmail.com
+
+LABEL vendor="Cancer Genome Project, Wellcome Trust Sanger Institute"
+LABEL uk.ac.sanger.cgp.description="CGP WGS pipeline for dockstore.org"
+LABEL uk.ac.sanger.cgp.version="2.0.0"
+
+RUN bash -c 'apt-get update -yq >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'apt-get install -qy --no-install-recommends lsb-release >& this.log || (cat this.log 1>&2 && exit 1)'
+
+RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" >> /etc/apt/sources.list
+RUN gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
+RUN gpg -a --export E084DAB9 | apt-key add -
+
+RUN bash -c 'apt-get update -yq >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'apt-get install -yq --no-install-recommends\
+  apt-transport-https\
+  locales\
+  curl\
+  ca-certificates\
+  libperlio-gzip-perl\
+  libssh2-1\
+  bzip2\
+  psmisc\
+  time\
+  zlib1g\
+  liblzma5\
+  libncurses5\
+  libcairo2\
+  gfortran\
+  r-base\
+  exonerate\
+  libboost-iostreams-dev\
+  p11-kit\
+     >& this.log || (cat this.log 1>&2 && exit 1)'
+
+RUN bash -c 'locale-gen en_US.UTF-8 >& this.log || (cat this.log 1>&2 && exit 1)'
+RUN bash -c 'update-locale LANG=en_US.UTF-8 >& this.log || (cat this.log 1>&2 && exit 1)'
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV R_LIBS $OPT/R-lib
+ENV R_LIBS_USER $R_LIBS
+
+RUN mkdir -p $OPT
+COPY --from=builder $OPT $OPT
+
 COPY scripts/analysisWGS.sh $OPT/bin/analysisWGS.sh
-COPY scripts/ds-wrapper.pl $OPT/bin/ds-wrapper.pl
-RUN chmod a+x $OPT/bin/analysisWGS.sh $OPT/bin/ds-wrapper.pl
+COPY scripts/ds-cgpwgs.pl $OPT/bin/ds-cgpwgs.pl
+RUN chmod a+x $OPT/bin/analysisWGS.sh $OPT/bin/ds-cgpwgs.pl
 
-### security upgrades and cleanup
-RUN apt-get -yq update && \
-    apt-get -yq install unattended-upgrades && \
-    unattended-upgrades
+## USER CONFIGURATION
+RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu
 
 USER    ubuntu
 WORKDIR /home/ubuntu