Mirroring integration #1
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Experiment | |
| permissions: read-all | |
| concurrency: | |
| group: ci-experiment-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - staging | |
| - trying | |
| pull_request: | |
| branches: [master] | |
| paths: | |
| - "**" | |
| - "!/*.md" | |
| - "!/**.md" | |
| - "!/rfc/**" | |
| workflow_dispatch: | |
| inputs: | |
| verbose: | |
| description: "Set --verbose to get verbose build output" | |
| required: false | |
| default: "" | |
| rust-channel: | |
| description: "Version of Rust release channel to use in builds" | |
| required: false | |
| type: choice | |
| options: | |
| - stable | |
| - beta | |
| - nightly | |
| default: stable | |
| build-name: | |
| description: "Name to add to VERSION. (git hash will be appended to build-name automatically)" | |
| required: true | |
| type: string | |
| env: | |
| CARGO_TERM_COLOR: always | |
| VERBOSE: ${{ github.events.input.verbose }} | |
| K3D_VERSION: v5.4.9 | |
| BATS_VERSION: 1.9.0 | |
| MINIKUBE_VERSION: v1.30.1 | |
| K8_VERSION: v1.26.3 | |
| TLS_ARGS: --tls --domain fluvio.local --server-key ./tls/certs/server.key --server-cert ./tls/certs/server.crt --ca-cert ./tls/certs/ca.crt --client-cert ./tls/certs/client-root.crt --client-key ./tls/certs/client-root.key | |
| AUTH_FILE: crates/fluvio-sc/test-data/auth_config/policy.json | |
| X509_SCOPE_FILE: crates/fluvio-sc/test-data/auth_config/scopes.json | |
| FLV_CLUSTER_PROVISION_TIMEOUT: 600 | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| # this job set up dynamic configuration shared among jobs | |
| config: | |
| name: Set up Config | |
| runs-on: ubuntu-latest | |
| env: | |
| ENABLE_CHECK: true | |
| TEST_RUNS: "[r1]" # default runs | |
| outputs: | |
| check: ${{ steps.check.outputs.check }} | |
| runs: ${{ steps.runs.outputs.runs }} | |
| steps: | |
| - id: check | |
| run: echo "::set-output name=check::${{ env.ENABLE_CHECK }}" | |
| build_binaries_for_docker: | |
| name: Build ${{ matrix.binary }} for ${{ matrix.rust-target }} on (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| # fail-fast: false | |
| matrix: | |
| include: | |
| # fluvio-run | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: aarch64-unknown-linux-musl | |
| binary: fluvio-run | |
| env: | |
| RUST_BACKTRACE: full | |
| RUSTV: ${{ matrix.rust }} | |
| TARGET: ${{ matrix.rust-target }} | |
| RUST_BIN_DIR: target/${{ matrix.rust-target }}/debug | |
| RELEASE_NAME: debug | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # If this job is being run by Bors (it was pushed to staging), | |
| # then build and run in release mode | |
| - name: Set RELEASE mode | |
| shell: bash | |
| run: | | |
| echo "RELEASE=true" | tee -a $GITHUB_ENV | |
| echo "RELEASE_NAME=release" | tee -a $GITHUB_ENV | |
| echo "RUST_BIN_DIR=target/${{ matrix.rust-target }}/release" | tee -a $GITHUB_ENV | |
| echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" | tee -a $GITHUB_ENV | |
| - name: Print env | |
| run: | | |
| echo "RUST_BIN_DIR = ${{ env.RUST_BIN_DIR }} " | |
| - name: Install Rust ${{ matrix.rust }} | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: ${{ matrix.rust }} | |
| profile: minimal | |
| override: true | |
| - name: Install LLVM and Clang | |
| if: contains(fromJSON('["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"]'), matrix.rust-target) | |
| uses: KyleMayes/install-llvm-action@v1 | |
| with: | |
| version: "14.0" | |
| - name: Install zig | |
| run: ./actions/zig-install.sh ${{ matrix.os }} | |
| - uses: Swatinem/rust-cache@v2 | |
| timeout-minutes: 10 | |
| with: | |
| key: ${{ matrix.os }}-${{ matrix.rust-target }}-${{ matrix.binary }} | |
| # If build name given, add to VERSION | |
| - name: Add experimental build name to VERSION | |
| #if: inputs.build-name | |
| run: |- | |
| echo FLUVIO_REPO_VERSION=$(cat VERSION) | tee -a $GITHUB_ENV | |
| echo -n "-${{ inputs.build-name }}-$(git rev-parse --short HEAD)" >> VERSION | |
| cat VERSION | wc | |
| cat VERSION | |
| - name: Build fluvio-run | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio-run' | |
| run: make build-cluster | |
| # Upload artifacts | |
| - name: Upload artifact - ${{ matrix.binary }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.binary }}-${{ matrix.rust-target }} | |
| path: ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }} | |
| retention-days: 1 | |
| # If this release does not already exist (i.e. check_fluvio failed), continue | |
| - name: Install fluvio-package | |
| env: | |
| CHANNEL_TAG: stable # We want to ensure we install the stable version of CLI | |
| run: | | |
| make curl-install-fluvio | |
| make install-fluvio-package | |
| - name: Publish artifacts | |
| env: | |
| FLUVIO_BIN: ~/.fluvio/bin/fluvio | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| ${{ env.FLUVIO_BIN }} package publish --package=${{ matrix.binary }} --version=${{ env.FLUVIO_REPO_VERSION }}+${{ inputs.build-name }}-$(git rev-parse --short HEAD) --target=${{ matrix.rust-target }} ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }} | |
| build_binaries: | |
| name: Build ${{ matrix.binary }} for ${{ matrix.rust-target }} on (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| # fail-fast: false | |
| matrix: | |
| include: | |
| # fluvio | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: aarch64-unknown-linux-musl | |
| binary: fluvio | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: x86_64-unknown-linux-musl | |
| binary: fluvio | |
| #- os: ubuntu-latest | |
| # rust: stable | |
| # rust-target: arm-unknown-linux-gnueabihf | |
| # binary: fluvio | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: armv7-unknown-linux-gnueabihf | |
| binary: fluvio | |
| #- os: ubuntu-latest | |
| # rust: stable | |
| # rust-target: x86_64-pc-windows-gnu | |
| # binary: fluvio.exe | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: x86_64-apple-darwin | |
| binary: fluvio | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: aarch64-apple-darwin | |
| binary: fluvio | |
| # fluvio-run | |
| # Note: aarch64-unknown-linux-musl target moved to separate job | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: x86_64-unknown-linux-musl | |
| binary: fluvio-run | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: armv7-unknown-linux-gnueabihf | |
| binary: fluvio-run | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: x86_64-apple-darwin | |
| binary: fluvio-run | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: aarch64-apple-darwin | |
| binary: fluvio-run | |
| # fluvio-channel | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: aarch64-unknown-linux-musl | |
| binary: fluvio-channel | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: x86_64-unknown-linux-musl | |
| binary: fluvio-channel | |
| #- os: ubuntu-latest | |
| # rust: stable | |
| # rust-target: arm-unknown-linux-gnueabihf | |
| # binary: fluvio-channel | |
| - os: ubuntu-latest | |
| rust: stable | |
| rust-target: armv7-unknown-linux-gnueabihf | |
| binary: fluvio-channel | |
| #- os: ubuntu-latest | |
| # rust: stable | |
| # rust-target: x86_64-pc-windows-gnu | |
| # binary: fluvio-channel.exe | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: x86_64-apple-darwin | |
| binary: fluvio-channel | |
| - os: macos-12 | |
| rust: stable | |
| rust-target: aarch64-apple-darwin | |
| binary: fluvio-channel | |
| env: | |
| RUST_BACKTRACE: full | |
| RUSTV: ${{ matrix.rust }} | |
| TARGET: ${{ matrix.rust-target }} | |
| RUST_BIN_DIR: target/${{ matrix.rust-target }}/debug | |
| RELEASE_NAME: debug | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set RELEASE mode | |
| shell: bash | |
| run: | | |
| echo "RELEASE=true" | tee -a $GITHUB_ENV | |
| echo "RELEASE_NAME=release" | tee -a $GITHUB_ENV | |
| echo "RUST_BIN_DIR=target/${{ matrix.rust-target }}/release" | tee -a $GITHUB_ENV | |
| - name: Print env | |
| run: | | |
| echo "RUST_BIN_DIR = ${{ env.RUST_BIN_DIR }} " | |
| - name: Install Rust ${{ matrix.rust }} | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: ${{ matrix.rust }} | |
| profile: minimal | |
| override: true | |
| - name: Install LLVM and Clang | |
| uses: KyleMayes/install-llvm-action@v1 | |
| if: contains(fromJSON('["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"]'), matrix.rust-target) | |
| with: | |
| version: "14.0" | |
| - name: Install zig | |
| run: ./actions/zig-install.sh ${{ matrix.os }} | |
| - uses: Swatinem/rust-cache@v2 | |
| timeout-minutes: 10 | |
| with: | |
| key: ${{ matrix.os }}-${{ matrix.rust-target }}-${{ matrix.binary }} | |
| - name: Install Helm for Mac | |
| run: brew install helm | |
| if: matrix.binary == 'fluvio' && matrix.os == 'macos-12' | |
| - name: Add experimental build name to VERSION | |
| run: |- | |
| echo FLUVIO_REPO_VERSION=$(cat VERSION) >> $GITHUB_ENV | |
| echo -n "-${{ inputs.build-name }}-$(git rev-parse --short HEAD)" >> VERSION | |
| cat VERSION | wc | |
| cat VERSION | |
| - name: Build fluvio | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio' | |
| run: make build-cli | |
| - name: Install mingw gcc | |
| if: matrix.rust-target == 'x86_64-pc-windows-gnu' | |
| run: | | |
| sudo apt-get update -o="APT::Acquire::Retries=3" | |
| sudo apt-get install -y -V -o="APT::Acquire::Retries=3" gcc-mingw-w64-x86-64 | |
| - name: Build fluvio.exe | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio.exe' | |
| run: make build-cli-minimal | |
| - name: Build fluvio-run | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio-run' | |
| run: make build-cluster | |
| - name: Build fluvio-channel | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio-channel' | |
| run: make build-channel | |
| - name: Build fluvio-channel.exe | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fluvio-channel.exe' | |
| run: make build-channel | |
| - name: Build smdk | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'smdk' | |
| run: make build-smdk | |
| - name: Build smdk.exe | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'smdk.exe' | |
| run: make build-smdk | |
| - name: Build cdk | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'cdk' | |
| run: make build-cdk | |
| - name: Build fbm | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fbm' | |
| run: make build-fbm | |
| - name: Build fbm.exe | |
| timeout-minutes: 40 | |
| if: matrix.binary == 'fbm.exe' | |
| run: make build-fbm | |
| # Upload artifacts | |
| - name: Upload artifact - ${{ matrix.binary }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.binary }}-${{ matrix.rust-target }} | |
| path: ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }} | |
| retention-days: 1 | |
| - name: Install fluvio-package | |
| env: | |
| CHANNEL_TAG: stable # We want to ensure we install the stable version of CLI | |
| run: | | |
| make curl-install-fluvio | |
| make install-fluvio-package | |
| - name: Publish artifacts | |
| env: | |
| FLUVIO_BIN: ~/.fluvio/bin/fluvio | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| ${{ env.FLUVIO_BIN }} package publish --package=${{ matrix.binary }} --version=${{ env.FLUVIO_REPO_VERSION }}+${{ inputs.build-name }}-$(git rev-parse --short HEAD) --target=${{ matrix.rust-target }} ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }} | |
| build_image: | |
| name: Build Fluvio Docker image | |
| needs: build_binaries_for_docker | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| rust-target: | |
| - aarch64-unknown-linux-musl | |
| rust: [stable] | |
| steps: | |
| # Needed for k8-util/docker/build.sh | |
| - uses: actions/checkout@v3 | |
| # If build name given, add to VERSION | |
| - name: Add experimental build name to VERSION | |
| run: |- | |
| echo FLUVIO_REPO_VERSION=$(cat VERSION) | tee -a $GITHUB_ENV | |
| echo -n "-${{ inputs.build-name }}-$(git rev-parse --short HEAD)" >> VERSION | |
| cat VERSION | wc | |
| cat VERSION | |
| echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" | tee -a $GITHUB_ENV | |
| # Download artifacts | |
| - name: Download fluvio-run | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: fluvio-run-${{ matrix.rust-target }} | |
| path: . | |
| - name: Print fluvio-run path | |
| run: pwd && ls -la . && chmod +x ./fluvio-run && ./fluvio-run -h || true | |
| # Build and upload docker image | |
| - name: Build Docker image | |
| run: k8-util/docker/build.sh ${{ matrix.rust-target }} ${{ github.sha }} "$(pwd)/fluvio-run" | |
| - name: Re-tag the docker image | |
| run: docker tag infinyon/fluvio:${{ github.sha }}-${{ matrix.rust-target }} infinyon/fluvio:${{ env.FLUVIO_REPO_VERSION }}-${{ inputs.build-name }}-${{ env.GIT_SHA_SHORT }} | |
| - name: Export Docker Image to tarball | |
| run: docker image save infinyon/fluvio:${{ github.sha }}-${{ matrix.rust-target }} --output infinyon-fluvio-${{ matrix.rust-target }}.tar | |
| - name: Upload tarball as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: infinyon-fluvio-${{ matrix.rust-target }}.tar | |
| path: infinyon-fluvio-${{ matrix.rust-target }}.tar | |
| retention-days: 1 | |
| - uses: keithweaver/aws-s3-github-action@v1.0.0 | |
| env: | |
| FLUVIO_REPO_VERSION: ${{ env.FLUVIO_REPO_VERSION }} | |
| GIT_SHA_SHORT: ${{ env.GIT_SHA_SHORT }} | |
| with: | |
| command: cp | |
| source: infinyon-fluvio-${{ matrix.rust-target }}.tar | |
| destination: s3://packages.fluvio.io/v1/images/infinyon-fluvio-${{ matrix.rust-target }}-${{ env.FLUVIO_REPO_VERSION }}-${{ inputs.build-name }}-${{ env.GIT_SHA_SHORT }}.tar | |
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-east-1 | |
| flags: --acl public-read | |
| # Job that follows the success of all required jobs in this workflow. | |
| # Used by Bors to detect that all required jobs have completed successfully | |
| done: | |
| name: Done | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/staging' | |
| needs: [build_binaries, build_image] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Done | |
| run: echo "Done!" |