From 58694133000429be45ce564c6a04a468d6a15cf2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 17 Nov 2024 17:44:27 -0500 Subject: [PATCH] update workflow files to use hatch.toml --- .github/scripts/update_dependencies.sh | 6 +- .../scripts/update_dev_dependency_branches.sh | 4 +- .github/workflows/integration.yml | 88 +--- .github/workflows/main.yml | 124 +---- .github/workflows/release.yml | 113 ++--- .github/workflows/release_prep_hatch.yml | 469 ++++++++++++++++++ pyproject.toml | 1 + 7 files changed, 555 insertions(+), 250 deletions(-) create mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh index c3df48e52..7880b0163 100644 --- a/.github/scripts/update_dependencies.sh +++ b/.github/scripts/update_dependencies.sh @@ -2,9 +2,9 @@ set -e git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g" +tests_req_sed_pattern="s|dbt-adapters.git#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh index 022df6a8a..c78029e5e 100755 --- a/.github/scripts/update_dev_dependency_branches.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -5,8 +5,8 @@ set -e dbt_adapters_branch=$1 dbt_core_branch=$2 dbt_common_branch=$3 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +target_req_file="hatch.toml" +core_req_sed_pattern="s|dbt-core.git#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g" adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b3662d5c0..1ff7c1149 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -86,23 +86,17 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate-matrix.outputs.result }} - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' + - if: github.event_name != 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' # https://github.com/marketplace/actions/paths-changes-filter # For each filter, it sets output variable named by the filter to the text: # 'true' - if any of changed files matches any of filter rules @@ -119,10 +113,9 @@ jobs: - '.github/**/*.sh' - 'dbt/**' - 'tests/**' - - 'dev-requirements.txt' + - 'pyproject.toml' - '*.py' - - name: Generate integration test matrix - id: generate-matrix + - id: generate-matrix uses: actions/github-script@v7 env: CHANGES: ${{ steps.get-changes.outputs.changes }} @@ -132,9 +125,9 @@ jobs: const matrix = script({ context }) console.log(matrix) return matrix + test: name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - # run if not a PR from a forked repository or has a label to mark as safe to test # also checks that the matrix generated is not empty if: >- @@ -147,67 +140,42 @@ jobs: ) runs-on: ${{ matrix.os }} needs: test-metadata - strategy: fail-fast: false matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_INVOCATION_ENV: github-actions DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} - steps: - - name: Check out the repository - if: github.event_name == 'push' + - if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false - - - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' + - if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ inputs.adapter_branch }} - - # explicitly checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' + - if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Update Adapters and Core branches (update dev_requirements.txt) - if: ${{ github.event_name == 'workflow_dispatch' }} + - if: ${{ github.event_name == 'workflow_dispatch' }} run: | ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} \ ${{ inputs.dbt_common_branch }} - cat dev-requirements.txt - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox (snowflake) - if: matrix.adapter == 'snowflake' + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: pypa/hatch@install + - run: hatch run integration-tests env: SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} @@ -226,38 +194,14 @@ jobs: DBT_TEST_USER_1: dbt_test_role_1 DBT_TEST_USER_2: dbt_test_role_2 DBT_TEST_USER_3: dbt_test_role_3 - run: tox -- --ddtrace - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }} - path: ./logs - overwrite: true - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - overwrite: true require-label-comment: runs-on: ubuntu-latest - needs: test - permissions: pull-requests: write - steps: - - name: Needs permission PR comment - if: >- + - if: >- needs.test.result == 'skipped' && github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24d2fa60b..d89f635d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,124 +38,52 @@ defaults: jobs: code-quality: name: code-quality - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - dbt --version - - - name: Run pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure + - uses: pre-commit/action@v3.0.1 unit: name: unit test / python ${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - overwrite: true + - uses: pypa/hatch@install + - run: hatch run unit-tests build: name: build packages - runs-on: ubuntu-latest - outputs: is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Check if this is an alpha version - id: check-is-alpha + - run: hatch build + - run: hatch run build:check-all + - id: check-is-alpha run: | export is_alpha=0 if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v4 with: name: dist @@ -164,44 +92,22 @@ jobs: test-build: name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} - if: needs.build.outputs.is_alpha == 0 - needs: build - runs-on: ${{ matrix.os }} - strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-14, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] dist-type: ['whl', 'gz'] - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - uses: actions/download-artifact@v4 with: name: dist path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install ${{ matrix.dist-type }} distributions - run: | - find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check ${{ matrix.dist-type }} distributions - run: | - python -c "import dbt.adapters.snowflake" + - run: find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + - run: python -c "import dbt.adapters.snowflake" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad7cf76b4..771228d73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,86 +119,71 @@ defaults: shell: bash jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Only Docker: ${{ inputs.only_docker }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + release-prep: + name: "Release prep: generate changelog, bump version" + uses: dbt-labs/dbt-snowflake/.github/workflows/release_prep_hatch.yml@main with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} steps: - - name: Print variables + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-prep.outputs.release-branch }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python_version }} + - uses: pypa/hatch@install + - id: archive run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + - run: hatch build && hatch run build:check-all + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.archive.outputs.name }} + path: dist/ + retention-days: 3 - build-test-package: - name: Build, Test, Package + pypi-release: + name: "PyPI release" if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + runs-on: ubuntu-latest + needs: build-release + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - name: "Publish to PyPI" + uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ needs.build-release.outputs.archive-name }} github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + needs: [build-release, release-prep] + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - if: ${{ !failure() && !cancelled() && !inputs.only_docker }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + archive_name: ${{ needs.build-release.outputs.archive-name }} docker-release: name: "Docker Release" @@ -206,7 +191,7 @@ jobs: # what we need to release but draft releases don't actually tag the commit so it # finds nothing to release if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} - needs: [bump-version-generate-changelog, build-test-package, github-release] + needs: github-release permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 000000000..18d7637b8 --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,469 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string + deploy-to: + description: "Deploy to test or prod" + type: string + default: "prod" + is-nightly-release: + description: "Identify if this is a nightly release" + type: boolean + default: false + outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.12 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_current" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + core-team: + if: needs.release-inputs.outputs.changelog-exists == 'false' + needs: release-inputs + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + secrets: inherit + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - core-team + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run unit tests" + run: hatch run unit-tests + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: Setup postgres + run: psql -f ./scripts/setup_test_database.sql + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + + - name: "Set up `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Run integration tests" + run: hatch run integration-tests + env: + POSTGRES_TEST_HOST: localhost + POSTGRES_TEST_PORT: 5432 + POSTGRES_TEST_USER: root + POSTGRES_TEST_PASS: password + POSTGRES_TEST_DATABASE: dbt + POSTGRES_TEST_THREADS: 4 + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.result == 'success' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.event.repository.name }}" + uses: actions/checkout@v4 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in + run: | + if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then + branch="${{ needs.release-branch.outputs.name }}" + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.branch.outputs.name }} + + - name: "Set release SHA" + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} diff --git a/pyproject.toml b/pyproject.toml index 577932f47..520e21a51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md" [tool.pytest] testpaths = ["tests/functional", "tests/unit"] +addopts = "-v -n auto" color = true filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning",