[feature] Count newlines during decompression to maket it work with n… #566
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: C++ Code Checks | |
| on: | |
| push: | |
| branches: [ "develop" ] | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| Code-Coverage: | |
| name: Code Coverage | |
| runs-on: ubuntu-22.04 | |
| # Suddenly, the upload to codecov does not work anymore and times out after 6 hours -.-. | |
| # It's not like I used it much anyway, except to have a badge. | |
| if: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get -y install bzip2 cppcheck clang clang-tidy g++ lcov ninja-build wget gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: System Information | |
| run: | | |
| nproc | |
| cmake --version | |
| g++ --version | |
| clang++ --version | |
| cppcheck --version | |
| - name: Code Coverage | |
| run: | | |
| mkdir -p build-codeov && cd -- "$_" | |
| # It should be compiled with the same g++ version as anylzed with gcov! | |
| # If g++ --version and gcov --version print different versions, there might be problems when creating | |
| # the coverage report, e.g., I had: "version 'A85*', prefer 'B12*'". | |
| # It is possible to specify a different gcov version to lcov using, e.g., lcov --gcov-tool gcov-8. | |
| # For me it compiled with g++-8 but used gcov-11 by default! | |
| CXX=g++ CC=gcc cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE=ON -DCMAKE_CXX_FLAGS="-DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| lcov --capture --directory . --base-directory "$( cd .. && pwd )" --no-external --output-file coverage.info | |
| lcov --list coverage.info | |
| bash <( wget -O- -q https://codecov.io/bash ) -f coverage.info | |
| - name: Run cppcheck | |
| # Crashes on templated variable REVERSED_BITS_LUT<T>[value] (internalAstError) | |
| if: false | |
| run: | | |
| set -o pipefail | |
| cppcheck -j $( nproc ) --enable=all --std=c++17 --suppress=unusedFunction --suppress=useStlAlgorithm \ | |
| -I src/core \ | |
| -I src/indexed_bzip2 \ | |
| -I src/rapidgzip \ | |
| -I src/rapidgzip/huffman \ | |
| src/benchmarks \ | |
| src/core \ | |
| src/indexed_bzip2 \ | |
| src/rapidgzip \ | |
| src/tests \ | |
| src/tools 2>&1 | tee cppcheck.log | |
| if grep -E -i 'error: .* \[[^]]*\]$' cppcheck.log; then | |
| echo -e '\e[31mThere were errors while running cppcheck!\e[0m' | |
| exit 1 | |
| fi | |
| Clang-Tidy: | |
| name: Clang-Tidy | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get -y install clang clang-tidy ninja-build libarchive-dev zlib1g-dev nasm | |
| - name: Run clang-tidy | |
| run: | | |
| mkdir -p build-compile-commands && cd -- "$_" | |
| CC=clang CXX=clang++ cmake -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_SYSTEM_ZLIB=ON -DRUN_CLANG_TIDY=ON .. | |
| cmake --build . | |
| Check: | |
| name: Check without Sanitizer | |
| # And especially without -DSHORT_TESTS, which makes it non redundant to the sanitizer checks! | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| use_system_zlib: [ON, OFF] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get -y install g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Sanitizers | |
| env: | |
| USE_SYSTEM_ZLIB: ${{ matrix.use_system_zlib }} | |
| run: | | |
| mkdir -p "build" && cd -- "$_" | |
| CXX=g++ CC=gcc cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DWITH_ISAL=ON -DUSE_SYSTEM_ZLIB=$USE_SYSTEM_ZLIB .. | |
| cmake --build . -- check | |
| Check-Without-Isal: | |
| name: Check without Sanitizer and without ISA-L | |
| # And especially without -DSHORT_TESTS, which makes it non redundant to the sanitizer checks! | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-12, macos-13, windows-latest] | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: "10.15" | |
| steps: | |
| - name: System Information | |
| if: "! startsWith(matrix.os, 'windows')" | |
| run: | | |
| uname -a | |
| - name: Print Architecture | |
| if: "startsWith(matrix.os, 'macos')" | |
| run: | | |
| sysctl -n machdep.cpu.brand_string | |
| arch | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: seanmiddleditch/gha-setup-ninja@master | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| brew install ninja pigz zlib isa-l wget | |
| wget 'https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2' | |
| tar -xf htslib-1.20.tar.bz2 | |
| cd htslib* && ./configure && make install | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sudo apt-get -y install g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal | |
| - name: Fix pip | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| mkdir -p "$HOME/Library/Application Support/pip" | |
| cat <<EOF >> "$HOME/Library/Application Support/pip/pip.conf" | |
| [global] | |
| break-system-packages = true | |
| EOF | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| mkdir -p "build" && cd -- "$_" | |
| cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DWITH_ISAL=OFF -DUSE_SYSTEM_ZLIB=OFF .. | |
| cmake --build . -- check | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| # For some reason MacOS ARM is much slower and runs into a timeout without -DSHORT_TESTS :/ | |
| mkdir -p "build" && cd -- "$_" | |
| cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DWITH_ISAL=OFF -DUSE_SYSTEM_ZLIB=OFF -DCMAKE_CXX_FLAGS="-DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'windows')" | |
| run: | | |
| mkdir -p "build" && cd -- "build" | |
| cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DWITH_ISAL=OFF .. | |
| cmake --build . -- check | |
| Asan: | |
| name: ASan | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-13] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| brew install ninja pigz zlib isa-l wget nasm | |
| wget 'https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2' | |
| tar -xf htslib-1.20.tar.bz2 | |
| cd htslib* && ./configure && make install | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sudo apt-get -y install clang g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| sanitizer=address | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| cmake -G Ninja -DWITH_ISAL=OFF -DUSE_SYSTEM_ZLIB=OFF -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sanitizer=address | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| CXX=clang++ CC=clang cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| cmake --build . | |
| bash ../src/tests/testRapidgzip.sh | |
| UBSan: | |
| name: UBSan | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-13] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| brew install ninja pigz zlib isa-l wget nasm | |
| wget 'https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2' | |
| tar -xf htslib-1.20.tar.bz2 | |
| cd htslib* && ./configure && make install | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sudo apt-get -y install clang g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| sanitizer=undefined | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| cmake -G Ninja -DWITH_ISAL=OFF -DUSE_SYSTEM_ZLIB=OFF -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- testParallelGzipReader | |
| src/tests/rapidgzip/testParallelGzipReader | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sanitizer=undefined | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| CXX=clang++ CC=clang cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| Safe-Stack: | |
| name: Safe-Stack | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-13] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| brew install ninja pigz zlib isa-l wget nasm gcc | |
| wget 'https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2' | |
| tar -xf htslib-1.20.tar.bz2 | |
| cd htslib* && ./configure && make install | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sudo apt-get -y install clang g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| sanitizer=address | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| CXX=g++ CC=gcc cmake -G Ninja -DWITH_ISAL=OFF -DUSE_SYSTEM_ZLIB=OFF -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- testParallelGzipReader | |
| src/tests/rapidgzip/testParallelGzipReader | |
| - name: Run Tests With Sanitizers | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sanitizer=safe-stack | |
| mkdir -p "build-sanitized-$sanitizer" && cd -- "$_" | |
| CXX=clang++ CC=clang cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=$sanitizer -DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| TSan: | |
| name: TSan | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get -y install clang g++ ninja-build gzip tabix pigz gztool libarchive-dev zlib1g-dev isal nasm | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade indexed_gzip pgzip | |
| - name: Run Tests With Thread Sanitizer | |
| run: | | |
| sanitized=thread | |
| mkdir -p "build-sanitized-$sanitized" && cd -- "$_" | |
| # | |
| # TSAN in clang 10 and 11 shows false positives in condition variables: | |
| # https://github.com/google/sanitizers/issues/1259 | |
| # Similar false positives appear with GCC 10. | |
| # NO warnings appear with: GCC 8, 9 and clang 12, | |
| # so I'm pretty sure that these are the referenced false positives. | |
| clangMajorVersion=$( clang++ --version | sed -n -E 's|.* ([0-9]+)[.][0-9]+[.][0-9]+.*|\1|p' | head -1 ) | |
| if [[ ( "$sanitized" == thread ) && ( "$clangMajorVersion" -le 11 ) ]]; then | |
| g++ --version | |
| gccMajorVersion=$( g++ --version | sed -n -E 's|.* ([0-9]+)[.][0-9]+[.][0-9]+.*|\1|p' | head -1 ) | |
| if [[ "$gccMajorVersion" -ge 10 ]]; then | |
| echo "Could not find a compiler version without known false positives. Skipping TSAN." | |
| continue | |
| fi | |
| CXX=g++ | |
| CC=gcc | |
| else | |
| CXX=clang++ | |
| CC=clang | |
| fi | |
| export CC CXX | |
| # | |
| cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=$sanitized -DSHORT_TESTS" .. | |
| cmake --build . -- check | |
| Code-Checks: | |
| name: Check with Infer | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get -y install ninja-build wget nasm | |
| # Install infer | |
| VERSION=1.1.0 | |
| wget -q -O- "https://github.com/facebook/infer/releases/download/v$VERSION/infer-linux64-v$VERSION.tar.xz" | tar -xJ | |
| echo "$PWD/infer-linux64-v$VERSION/bin" >> $GITHUB_PATH | |
| - name: Run infer | |
| run: | | |
| mkdir -p build-compile-commands && cd -- "$_" | |
| CC=clang CXX=clang++ cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON | |
| infer run --report-blacklist-path-regex '.*testParallelBitStringFinder.cpp' --compilation-database compile_commands.json | |
| Ibzip2-Tests: | |
| name: Ibzip2 Tests | |
| # And especially without -DSHORT_TESTS, which makes it non redundant to the sanitizer checks! | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-13, macos-14, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: seanmiddleditch/gha-setup-ninja@master | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| brew install ninja | |
| - name: Install System Dependencies | |
| if: "startsWith(matrix.os , 'ubuntu')" | |
| run: | | |
| sudo apt-get -y install g++ ninja-build | |
| - name: Fix pip | |
| if: "startsWith(matrix.os , 'macos')" | |
| run: | | |
| mkdir -p "$HOME/Library/Application Support/pip" | |
| cat <<EOF >> "$HOME/Library/Application Support/pip/pip.conf" | |
| [global] | |
| break-system-packages = true | |
| EOF | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install build | |
| - name: Install indexed_bzip2 | |
| working-directory: python/indexed_bzip2 | |
| shell: bash | |
| run: | | |
| python3 -m build | |
| python3 -m pip install dist/*.tar.gz | |
| - name: Run Standard Tests | |
| shell: bash | |
| # At this step, indexed_bzip2 should be installed. | |
| run: | | |
| mkdir -p build-ibzip2 && cd -- "$_" | |
| cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DWITH_ISAL=OFF .. | |
| cmake --build . -- ibzip2 | |
| set -o pipefail | |
| bash ../src/tests/testIbzip2.sh | tee run-standard-tests.log | |
| if grep -i 'failed' run-standard-tests.log; then | |
| echo -e '\e[31mThere were errors while running the tests!\e[0m' | |
| exit 1 | |
| fi |