Add workaround for snakemake logging bug #91
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ================================================================================================== | |
| # Header | |
| # ================================================================================================== | |
| name: CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| - dev | |
| # ================================================================================================== | |
| # Jobs | |
| # ================================================================================================== | |
| jobs: | |
| # ========================================================================== | |
| # test | |
| # ========================================================================== | |
| test: | |
| # if: ${{ false }} # disable for now | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| # Need to specify the shell explicitly. | |
| # https://github.com/marketplace/actions/setup-miniconda#important | |
| # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell | |
| shell: bash -el {0} | |
| # ------------------------------------------------------- | |
| # Matrix | |
| # ------------------------------------------------------- | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| # - macos-latest | |
| # - ubuntu-20.04 | |
| # - ubuntu-22.04 | |
| # - ubuntu-24.04 | |
| # - macos-12 | |
| # - macos-13 | |
| # - macos-14 | |
| # Test cases as listed in test/cases/*.txt | |
| # We update them automatically when running the test/run.sh script, so that each time | |
| # we have and run a new test case, it also appears here. The script to run this update | |
| # is located in .github/workflows/update-cases.sh | |
| case: | |
| # CASES_BEGIN | |
| - base | |
| - calling-bcftools-1 | |
| - calling-bcftools-2 | |
| - calling-bcftools-3 | |
| - calling-bcftools-4 | |
| - calling-freebayes-1 | |
| - calling-freebayes-2 | |
| - calling-freebayes-3 | |
| - calling-haplotypecaller-1 | |
| - calling-haplotypecaller-2 | |
| - calling-haplotypecaller-3 | |
| - calling-haplotypecaller-4 | |
| - download | |
| - duplicates-dedup-1 | |
| - duplicates-dedup-2 | |
| - hafpipe-1 | |
| - hafpipe-2 | |
| - hafpipe-3 | |
| - hafpipe-4 | |
| - mapping-bowtie2-1 | |
| - mapping-bowtie2-2 | |
| - mapping-bwa-aln-1 | |
| - mapping-bwa-aln-2 | |
| - mapping-bwa-aln-3 | |
| - mapping-bwa-aln-4 | |
| - mapping-bwa-mem-1 | |
| - mapping-bwa-mem2-1 | |
| - mapping-bwa-mem2-2 | |
| - mappings-table | |
| - pileup-1 | |
| - trimming-adapterremoval-1 | |
| - trimming-adapterremoval-2 | |
| - trimming-adapterremoval-3 | |
| - trimming-cutadapt-1 | |
| - trimming-cutadapt-2 | |
| - trimming-fastp-1 | |
| - trimming-fastp-2 | |
| - trimming-fastp-3 | |
| - trimming-none | |
| - trimming-seqprep-1 | |
| - trimming-seqprep-2 | |
| - trimming-skewer-1 | |
| - trimming-skewer-2 | |
| # CASES_END | |
| installer: | |
| # - conda | |
| - mamba | |
| include: | |
| # Also test with other operating systems. | |
| # For now, we just test the base case to see if it's generally working. | |
| # This repeats one of the Ubuntu runs for the base case, but that's okay | |
| # to keep it future proof when the default ubuntu-latest changes. | |
| - os: ubuntu-20.04 | |
| case: base | |
| - os: ubuntu-22.04 | |
| case: base | |
| - os: ubuntu-24.04 | |
| case: base | |
| - os: macos-12 | |
| case: base | |
| - os: macos-13 | |
| case: base | |
| # - os: macos-14 | |
| # case: base | |
| # ------------------------------------------------------- | |
| # Steps | |
| # ------------------------------------------------------- | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Conda | |
| uses: conda-incubator/setup-miniconda@v3.0.4 | |
| with: | |
| # We need mamba here already, otherwise we run into | |
| # https://github.com/conda-incubator/setup-miniconda/issues/262 | |
| # mamba-version: "*" | |
| # python-version: 3.12 | |
| # channels: conda-forge,defaults | |
| # channel-priority: true | |
| # use-mamba: true | |
| # environment-file: workflow/envs/grenepipe.yaml | |
| # activate-environment: grenepipe | |
| # We might need to replace the above, at least for now, due to instabilities in the action, | |
| # see https://github.com/conda-incubator/setup-miniconda/issues/274#issue-1531425010 | |
| # If that changes again in the future, also change it below in the other jobs. | |
| # miniforge-variant: Mambaforge | |
| # miniforge-version: latest | |
| # New workaround suggestion: | |
| # https://github.com/conda-incubator/setup-miniconda/issues/292#issuecomment-1767669652 | |
| # Why is this stuff so broken all the time? | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| use-mamba: true | |
| channels: conda-forge,defaults | |
| environment-file: workflow/envs/grenepipe.yaml | |
| activate-environment: grenepipe | |
| # Manually create the conda env. | |
| # Not needed at the moment, as the above setup works (if it works). | |
| # Keeping this here for reference. | |
| # - name: Install Conda Env | |
| # shell: bash -el {0} | |
| # run: | | |
| # mamba env create -f workflow/envs/grenepipe.yaml -n grenepipe | |
| # Then in the run step: | |
| # conda activate grenepipe | |
| # Run the test case. | |
| - name: Run Test | |
| run: | | |
| conda info | |
| mamba info | |
| if [[ "${{ matrix.installer }}" == "conda" ]]; then | |
| export GRENEPIPE_TESTS_NO_MAMBA=1 | |
| fi | |
| ./test/run.sh ${{ matrix.case }} | |
| # Store a list of all files and sizes, for debugging. | |
| - name: File Inventory | |
| if: always() | |
| run: | | |
| mkdir -p test/out-${{ matrix.case }}/logs | |
| ls -alFR test/out-${{ matrix.case }} > test/out-${{ matrix.case }}/logs/files.txt | |
| ls -alFR test/reference >> test/out-${{ matrix.case }}/logs/files.txt | |
| # Upload the log files as a GitHub Actions artifact | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-test-${{ matrix.os }}-${{ matrix.case }} | |
| path: | | |
| test/out-${{ matrix.case }}/config.yaml | |
| test/out-${{ matrix.case }}/test-run.log | |
| test/out-${{ matrix.case }}/logs/ | |
| test/out-${{ matrix.case }}/.snakemake/log/ | |
| test/reference/logs/ | |
| test/reference/TAIR10_chr_all.dict | |
| # ========================================================================== | |
| # example | |
| # ========================================================================== | |
| example: | |
| # if: ${{ false }} # disable for now | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| # ------------------------------------------------------- | |
| # Matrix | |
| # ------------------------------------------------------- | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-20.04 | |
| - ubuntu-22.04 | |
| - ubuntu-24.04 | |
| - macos-12 | |
| - macos-13 | |
| # - macos-14 | |
| # ------------------------------------------------------- | |
| # Steps | |
| # ------------------------------------------------------- | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Conda | |
| uses: conda-incubator/setup-miniconda@v3.0.3 | |
| with: | |
| # mamba-version: "*" | |
| # channels: conda-forge | |
| # Need to use a different setup to get mamba to work, see above. | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| use-mamba: true | |
| channels: conda-forge,defaults | |
| environment-file: workflow/envs/grenepipe.yaml | |
| activate-environment: grenepipe | |
| - name: Run Example | |
| run: | | |
| ./example/prepare.sh | |
| snakemake --use-conda --conda-frontend mamba --cores 4 --directory example/ 2>&1 | tee example/run.log | |
| - name: Upload Logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-example-${{ matrix.os }} | |
| path: | | |
| example/run.log | |
| example/logs/ | |
| example/.snakemake/log/ | |
| # ========================================================================== | |
| # envs | |
| # ========================================================================== | |
| envs: | |
| # if: ${{ false }} # disable for now | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| # ------------------------------------------------------- | |
| # Matrix | |
| # ------------------------------------------------------- | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-20.04 | |
| - ubuntu-22.04 | |
| - ubuntu-24.04 | |
| - macos-12 | |
| - macos-13 | |
| # - macos-14 | |
| installer: | |
| # Conda is just waaaay to slow... some runs even exceed the GitHub Actions | |
| # max runtime of 6h, so we have to deactivate it for now. | |
| # - conda | |
| - mamba | |
| - micromamba | |
| # ------------------------------------------------------- | |
| # Steps | |
| # ------------------------------------------------------- | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Conda | |
| uses: mamba-org/setup-micromamba@v1.9.0 | |
| with: | |
| # Trying another installer, see if that one finally works, | |
| # and to test if both mamba and micromamba work for all envs. | |
| # Any version from https://github.com/mamba-org/micromamba-releases | |
| micromamba-version: '1.5.8-0' | |
| environment-file: workflow/envs/grenepipe.yaml | |
| environment-name: grenepipe | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Install Envs | |
| shell: bash -el {0} | |
| run: | | |
| cd workflow/envs | |
| for env in `ls *.yaml` ; do | |
| name=${env%.yaml} | |
| printf "\n" ; printf '=%.0s' {1..100} | |
| printf "\n ${name}\n" | |
| printf '=%.0s' {1..100} ; printf "\n\n" | |
| if [[ "$name" = "grenepipe" ]]; then | |
| echo "Skipping grenepipe" | |
| continue | |
| fi | |
| if [[ ${{ matrix.os }} == macos* ]] && [[ "${name}" == *-linux ]]; then | |
| echo "Skipping linux env on macos" | |
| continue | |
| fi | |
| if [[ ${{ matrix.os }} == ubuntu* ]] && [[ "${name}" == *-macos ]]; then | |
| echo "Skipping macos env on linux" | |
| continue | |
| fi | |
| ${{ matrix.installer }} env create -f ${env} -n ${name} | |
| done |