dev-build.yml

# Development build only build the debug version of the artifacts manually.
name: GreptimeDB Development Build

on:
  workflow_dispatch: # Allows you to run this workflow manually.
    inputs:
      repository:
        description: The public repository to build
        required: false
        default: GreptimeTeam/greptimedb
      commit: # Note: We only pull the source code and use the current workflow to build the artifacts.
        description: The commit to build
        required: true
      linux_amd64_runner:
        type: choice
        description: The runner uses to build linux-amd64 artifacts
        default: ec2-c6i.4xlarge-amd64
        options:
          - ubuntu-20.04
          - ubuntu-20.04-8-cores
          - ubuntu-20.04-16-cores
          - ubuntu-20.04-32-cores
          - ubuntu-20.04-64-cores
          - ec2-c6i.xlarge-amd64 # 4C8G
          - ec2-c6i.2xlarge-amd64 # 8C16G
          - ec2-c6i.4xlarge-amd64 # 16C32G
          - ec2-c6i.8xlarge-amd64 # 32C64G
          - ec2-c6i.16xlarge-amd64 # 64C128G
      linux_arm64_runner:
        type: choice
        description: The runner uses to build linux-arm64 artifacts
        default: ec2-c6g.4xlarge-arm64
        options:
          - ec2-c6g.xlarge-arm64 # 4C8G
          - ec2-c6g.2xlarge-arm64 # 8C16G
          - ec2-c6g.4xlarge-arm64 # 16C32G
          - ec2-c6g.8xlarge-arm64 # 32C64G
          - ec2-c6g.16xlarge-arm64 # 64C128G
      skip_test:
        description: Do not run integration tests during the build
        type: boolean
        default: true
      build_linux_amd64_artifacts:
        type: boolean
        description: Build linux-amd64 artifacts
        required: false
        default: true
      build_linux_arm64_artifacts:
        type: boolean
        description: Build linux-arm64 artifacts
        required: false
        default: true
      release_images:
        type: boolean
        description: Build and push images to DockerHub and ACR
        required: false
        default: true
      cargo_profile:
        type: choice
        description: The cargo profile to use in building GreptimeDB.
        default: nightly
        options:
          - dev
          - release
          - nightly

# Use env variables to control all the release process.
env:
  CARGO_PROFILE: ${{ inputs.cargo_profile }}

  # Controls whether to run tests, include unit-test, integration-test and sqlness.
  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}

  # Always use 'dev' to indicate it's the dev build.
  NEXT_RELEASE_VERSION: dev

  NIGHTLY_RELEASE_PREFIX: nightly

  # Use the different image name to avoid conflict with the release images.
  IMAGE_NAME: greptimedb-dev

  # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'.
  CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb

permissions:
  issues: write

jobs:
  allocate-runners:
    name: Allocate runners
    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
    runs-on: ubuntu-20.04
    outputs:
      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}

      # The following EC2 resource id will be used for resource releasing.
      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}

      # The 'version' use as the global tag name of the release workflow.
      version: ${{ steps.create-version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Create version
        id: create-version
        run: |
          version=$(./.github/scripts/create-version.sh) && \
          echo $version && \
          echo "version=$version" >> $GITHUB_OUTPUT
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF_NAME: ${{ github.ref_name }}
          COMMIT_SHA: ${{ inputs.commit }}
          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}

      - name: Allocate linux-amd64 runner
        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
        uses: ./.github/actions/start-runner
        id: start-linux-amd64-runner
        with:
          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}

      - name: Allocate linux-arm64 runner
        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
        uses: ./.github/actions/start-runner
        id: start-linux-arm64-runner
        with:
          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}

  build-linux-amd64-artifacts:
    name: Build linux-amd64 artifacts
    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
    ]
    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Checkout greptimedb
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.repository }}
          ref: ${{ inputs.commit }}
          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}

      - uses: ./.github/actions/build-linux-artifacts
        with:
          arch: amd64
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
    ]
    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Checkout greptimedb
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.repository }}
          ref: ${{ inputs.commit }}
          path: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}

      - uses: ./.github/actions/build-linux-artifacts
        with:
          arch: arm64
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
      build-linux-arm64-artifacts,
    ]
    runs-on: ubuntu-20.04
    outputs:
      build-result: ${{ steps.set-build-result.outputs.build-result }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Build and push images to dockerhub
        uses: ./.github/actions/build-images
        with:
          image-registry: docker.io
          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
          image-name: ${{ env.IMAGE_NAME }}
          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
          version: ${{ needs.allocate-runners.outputs.version }}
          push-latest-tag: false # Don't push the latest tag to registry.
          dev-mode: true # Only build the standard images.

      - name: Set build result
        id: set-build-result
        run: |
          echo "build-result=success" >> $GITHUB_OUTPUT

  release-cn-artifacts:
    name: Release artifacts to CN region
    if: ${{ inputs.release_images || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
      release-images-to-dockerhub,
    ]
    runs-on: ubuntu-20.04
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Release artifacts to CN region
        uses: ./.github/actions/release-cn-artifacts
        with:
          src-image-registry: docker.io
          src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
          src-image-name: ${{ env.IMAGE_NAME }}
          dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
          dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
          dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
          dst-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          aws-cn-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
          aws-cn-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
          aws-cn-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
          aws-cn-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
          dev-mode: true                     # Only build the standard images(exclude centos images).
          push-latest-tag: false             # Don't push the latest tag to registry.
          update-version-info: false         # Don't update the version info in S3.

  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
    name: Stop linux-amd64 runner
    # Only run this job when the runner is allocated.
    if: ${{ always() }}
    runs-on: ubuntu-20.04
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
    ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Stop EC2 runner
        uses: ./.github/actions/stop-runner
        with:
          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
    name: Stop linux-arm64 runner
    # Only run this job when the runner is allocated.
    if: ${{ always() }}
    runs-on: ubuntu-20.04
    needs: [
      allocate-runners,
      build-linux-arm64-artifacts,
    ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Stop EC2 runner
        uses: ./.github/actions/stop-runner
        with:
          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

  notification:
    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run.
    name: Send notification to Greptime team
    needs: [
      release-images-to-dockerhub
    ]
    runs-on: ubuntu-20.04
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup-cyborg
      - name: Report CI status
        id: report-ci-status
        working-directory: cyborg
        run: pnpm tsx bin/report-ci-failure.ts
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
      - name: Notify dev build successful result
        uses: slackapi/slack-github-action@v1.23.0
        if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }}
        with:
          payload: |
            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has completed successfully."}

      - name: Notify dev build failed result
        uses: slackapi/slack-github-action@v1.23.0
        if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }}
        with:
          payload: |
            {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."}