diff --git a/.github/actions/install-dct-key/action.yml b/.github/actions/install-dct-key/action.yml new file mode 100644 index 000000000..85a281bc2 --- /dev/null +++ b/.github/actions/install-dct-key/action.yml @@ -0,0 +1,22 @@ +name: Install DCT Key +description: Installs a key file to use with DCT into the local DCT store. + +inputs: + key-id: + description: 'The ID of the key' + required: true + key-content: + description: 'The content of the key' + required: true + +runs: + using: "composite" + steps: + - name: Install DCT key + shell: bash + run: | + DELEGATION_KEY_FILE="$HOME/.docker/trust/private/${{ inputs.key-id }}.key" + mkdir -p $(dirname "$DELEGATION_KEY_FILE") + touch "$DELEGATION_KEY_FILE" + chmod 600 "$DELEGATION_KEY_FILE" + echo "${{ inputs.key-content }}" > "$DELEGATION_KEY_FILE" \ No newline at end of file diff --git a/.github/workflows/dct_pull_cd_preprod.yml b/.github/workflows/dct_pull_cd_preprod.yml new file mode 100644 index 000000000..67ceb88b7 --- /dev/null +++ b/.github/workflows/dct_pull_cd_preprod.yml @@ -0,0 +1,61 @@ +name: "CD: dct_pull (pre-prod)" + +on: + push: + branches: + - 'deploy-dct-pull-preprod' + +env: + TAG_VERSION: "v0-preprod" + STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor-staging" + DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor" + DIST_VERSION_PREFIX: "executor-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image + run: | + IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" + SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" + + docker pull "${SHA_IMAGE_NAME}" + docker image tag "${SHA_IMAGE_NAME}" "${IMAGE_NAME}" + + echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin + + docker push "${IMAGE_NAME}" + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Set Dist Version + run: | + BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + + - name: Create Tag + uses: actions/github-script@v6 + with: + script: | + const {DIST_VERSION} = process.env + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${DIST_VERSION}`, + sha: context.sha + }) diff --git a/.github/workflows/dct_pull_cd_prod.yml b/.github/workflows/dct_pull_cd_prod.yml new file mode 100644 index 000000000..deffaaea1 --- /dev/null +++ b/.github/workflows/dct_pull_cd_prod.yml @@ -0,0 +1,61 @@ +name: "CD: dct_pull (prod)" + +on: + push: + branches: + - 'deploy-dct-pull-prod' + +env: + TAG_VERSION: "v0-latest" + STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-dct-pull-staging" + DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-dct-pull" + DIST_VERSION_PREFIX: "dct-pull-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image + run: | + IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" + SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" + + docker pull "${SHA_IMAGE_NAME}" + docker image tag "${SHA_IMAGE_NAME}" "${IMAGE_NAME}" + + echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin + + docker push "${IMAGE_NAME}" + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Set Dist Version + run: | + BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + + - name: Create Tag + uses: actions/github-script@v6 + with: + script: | + const {DIST_VERSION} = process.env + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${DIST_VERSION}`, + sha: context.sha + }) diff --git a/.github/workflows/dct_pull_cd_staging.yml b/.github/workflows/dct_pull_cd_staging.yml new file mode 100644 index 000000000..1e4b22d22 --- /dev/null +++ b/.github/workflows/dct_pull_cd_staging.yml @@ -0,0 +1,63 @@ +name: "CD: dct_pull (staging)" + +on: + push: + branches: + - 'deploy-dct-pull-staging' + +env: + TAG_VERSION: "v0-latest" + BUILD_DIRECTORY: "dct_pull" + DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-dct-pull-staging" + DIST_VERSION_PREFIX: "dct-pull-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push + run: | + IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" + SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" + + cd "${BUILD_DIRECTORY}" && docker build --build-context compute-horde=../compute_horde -t "${IMAGE_NAME}" -f app/envs/prod/Dockerfile . + + docker image tag "${IMAGE_NAME}" "${SHA_IMAGE_NAME}" + + echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin + + docker push "${IMAGE_NAME}" + docker push "${SHA_IMAGE_NAME}" + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Set Dist Version + run: | + BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + + - name: Create Tag + uses: actions/github-script@v6 + with: + script: | + const {DIST_VERSION} = process.env + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${DIST_VERSION}`, + sha: context.sha + }) diff --git a/.github/workflows/executor_cd_preprod.yml b/.github/workflows/executor_cd_preprod.yml index 8d980c245..f5c4ebf5c 100644 --- a/.github/workflows/executor_cd_preprod.yml +++ b/.github/workflows/executor_cd_preprod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor" DIST_VERSION_PREFIX: "executor-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/executor_cd_prod.yml b/.github/workflows/executor_cd_prod.yml index 9bd32a1da..597464682 100644 --- a/.github/workflows/executor_cd_prod.yml +++ b/.github/workflows/executor_cd_prod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor" DIST_VERSION_PREFIX: "executor-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/executor_cd_staging.yml b/.github/workflows/executor_cd_staging.yml index d3585b874..2525913fd 100644 --- a/.github/workflows/executor_cd_staging.yml +++ b/.github/workflows/executor_cd_staging.yml @@ -10,6 +10,8 @@ env: TAG_VERSION: "v0-latest" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-executor-staging" DIST_VERSION_PREFIX: "executor-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -31,13 +40,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/executor_ci.yml b/.github/workflows/executor_ci.yml index ae94fe6d0..2951b200e 100644 --- a/.github/workflows/executor_ci.yml +++ b/.github/workflows/executor_ci.yml @@ -2,9 +2,9 @@ name: Run Executor linter and tests on: push: - branches: [master, main] + branches: [ master, main ] pull_request: - branches: [master, main] + branches: [ master, main ] env: PYTHON_DEFAULT_VERSION: "3.11" @@ -19,15 +19,19 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Run linters run: nox -vs lint + test: timeout-minutes: 10 runs-on: ubuntu-latest @@ -38,13 +42,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Setup common virtualenv # In order not to exhaust disk on GitHub runner, we use one single # virtualenv for all pdm projects: miner, executor, validator. @@ -54,10 +61,13 @@ jobs: echo "$.venv/bin" >> $GITHUB_PATH echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> $GITHUB_ENV echo "PDM_IGNORE_SAVED_PYTHON=1" >> $GITHUB_ENV + - name: Prepare environment run: ./setup-dev.sh + - name: Build backenddevelopersltd/compute-horde-job-echo:v0-latest image run: docker build . -t backenddevelopersltd/compute-horde-job-echo:v0-latest working-directory: ./executor/app/src/compute_horde_executor/executor/tests/integration/docker_image_for_tests + - name: Run unit tests run: pdm run nox -vs test --no-venv diff --git a/.github/workflows/integration_ci.yml b/.github/workflows/integration_ci.yml index 9cddc0e5d..ebbfa9c71 100644 --- a/.github/workflows/integration_ci.yml +++ b/.github/workflows/integration_ci.yml @@ -2,9 +2,9 @@ name: Run integration tests on: push: - branches: [master, main] + branches: [ master, main ] pull_request: - branches: [master, main] + branches: [ master, main ] env: PYTHON_DEFAULT_VERSION: "3.11" @@ -17,13 +17,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Setup common virtualenv # In order not to exhaust disk on GitHub runner, we use one single # virtualenv for all pdm projects: miner, executor, validator. @@ -33,34 +36,45 @@ jobs: echo "$.venv/bin" >> $GITHUB_PATH echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> $GITHUB_ENV echo "PDM_IGNORE_SAVED_PYTHON=1" >> $GITHUB_ENV + - name: Install test dependencies run: pdm install --check + - name: Prepare miner environment run: ./setup-dev.sh working-directory: ./miner + - name: Prepare executor environment run: ./setup-dev.sh working-directory: ./executor + - name: Prepare validator environment run: ./setup-dev.sh working-directory: ./validator + - name: Run miner dockerized services run: docker compose up -d --wait working-directory: ./miner + - name: Run validator dockerized services run: docker compose up -d --wait working-directory: ./validator + - name: Run miner migrations run: cd app/src && pdm run python manage.py wait_for_database --timeout 120 && pdm run python manage.py migrate working-directory: ./miner + - name: Run validator migrations run: cd app/src && pdm run python manage.py wait_for_database --timeout 120 && pdm run python manage.py migrate working-directory: ./validator + - name: Build backenddevelopersltd/compute-horde-job-echo:v0-latest image run: docker build . -t backenddevelopersltd/compute-horde-job-echo:v0-latest working-directory: ./executor/app/src/compute_horde_executor/executor/tests/integration/docker_image_for_tests/ + - name: Run unit tests run: pdm run nox -vs test --no-venv + - name: Stop miner dockerized services if: success() || failure() run: docker compose down -v diff --git a/.github/workflows/library_cd.yml b/.github/workflows/library_cd.yml index 518f42b82..f0a4bb77c 100644 --- a/.github/workflows/library_cd.yml +++ b/.github/workflows/library_cd.yml @@ -22,23 +22,29 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Get version from tag id: get-version run: echo "version=${GITHUB_REF#refs/tags/library-v}" >> "$GITHUB_OUTPUT" + # - name: Read the Changelog # id: read-changelog # uses: mindsers/changelog-reader-action@v2 # with: # version: ${{ steps.get-version.outputs.version }} # path: ./compute_horde/CHANGELOG.md + - name: Build run: pdm build + - name: Create GitHub release and upload the distribution id: create-release uses: softprops/action-gh-release@v1 @@ -48,6 +54,7 @@ jobs: draft: ${{ env.ACTIONS_STEP_DEBUG == 'true' }} prerelease: false files: ./compute_horde/dist/* + - name: Upload the distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/library_ci.yml b/.github/workflows/library_ci.yml index 375f9c6e1..ee54e81d8 100644 --- a/.github/workflows/library_ci.yml +++ b/.github/workflows/library_ci.yml @@ -2,9 +2,9 @@ name: Run library linter and tests on: push: - branches: [master, main] + branches: [ master, main ] pull_request: - branches: [master, main] + branches: [ master, main ] env: PYTHON_DEFAULT_VERSION: "3.11" @@ -19,15 +19,19 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Run linters run: nox -vs lint + test: timeout-minutes: 10 runs-on: ubuntu-latest @@ -38,12 +42,15 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Run unit tests run: nox -vs test diff --git a/.github/workflows/miner_cd_preprod.yml b/.github/workflows/miner_cd_preprod.yml index aa669ef9a..00cd9c58c 100644 --- a/.github/workflows/miner_cd_preprod.yml +++ b/.github/workflows/miner_cd_preprod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-miner-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-miner-preprod" DIST_VERSION_PREFIX: "miner-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/miner_cd_prod.yml b/.github/workflows/miner_cd_prod.yml index 7c583578b..2d6ba42f9 100644 --- a/.github/workflows/miner_cd_prod.yml +++ b/.github/workflows/miner_cd_prod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-miner-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-miner" DIST_VERSION_PREFIX: "miner-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/miner_cd_staging.yml b/.github/workflows/miner_cd_staging.yml index 7536f4d19..55291afe9 100644 --- a/.github/workflows/miner_cd_staging.yml +++ b/.github/workflows/miner_cd_staging.yml @@ -10,6 +10,8 @@ env: TAG_VERSION: "v0-latest" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-miner-staging" DIST_VERSION_PREFIX: "miner-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -36,13 +45,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/miner_ci.yml b/.github/workflows/miner_ci.yml index 84d824802..dfb17001c 100644 --- a/.github/workflows/miner_ci.yml +++ b/.github/workflows/miner_ci.yml @@ -2,9 +2,9 @@ name: Run Miner linter and tests on: push: - branches: [master, main] + branches: [ master, main ] pull_request: - branches: [master, main] + branches: [ master, main ] env: PYTHON_DEFAULT_VERSION: "3.11" @@ -19,15 +19,19 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Run linters run: nox -vs lint + test: timeout-minutes: 10 runs-on: ubuntu-latest @@ -38,13 +42,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Setup common virtualenv # In order not to exhaust disk on GitHub runner, we use one single # virtualenv for all pdm projects: miner, executor, validator. @@ -54,14 +61,19 @@ jobs: echo "$.venv/bin" >> $GITHUB_PATH echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> $GITHUB_ENV echo "PDM_IGNORE_SAVED_PYTHON=1" >> $GITHUB_ENV + - name: Prepare environment run: ./setup-dev.sh + - name: Run dockerized services run: docker compose up -d --wait + - name: Run migrations run: cd app/src && pdm run python manage.py wait_for_database --timeout 120 && pdm run python manage.py migrate + - name: Run unit tests run: pdm run nox -vs test --no-venv + - name: Stop dockerized services if: success() || failure() run: docker compose down -v diff --git a/.github/workflows/miner_runner_cd_preprod.yml b/.github/workflows/miner_runner_cd_preprod.yml index 19d33b458..fbe896938 100644 --- a/.github/workflows/miner_runner_cd_preprod.yml +++ b/.github/workflows/miner_runner_cd_preprod.yml @@ -13,6 +13,8 @@ env: MINER_IMAGE_REPO: "compute-horde-miner-preprod" MINER_NGINX_IMAGE_REPO: "compute-horde-miner-nginx-preprod" DIST_VERSION_PREFIX: "miner-runner-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -21,7 +23,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -40,7 +49,8 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" - - name: Docker build and push (nginx) + + - name: Docker build, sign and push (nginx) run: | IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:git-${GITHUB_SHA}" @@ -56,13 +66,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/miner_runner_cd_prod.yml b/.github/workflows/miner_runner_cd_prod.yml index f9f91b964..92e348990 100644 --- a/.github/workflows/miner_runner_cd_prod.yml +++ b/.github/workflows/miner_runner_cd_prod.yml @@ -13,6 +13,8 @@ env: MINER_IMAGE_REPO: "compute-horde-miner" MINER_NGINX_IMAGE_REPO: "compute-horde-miner-nginx" DIST_VERSION_PREFIX: "miner-runner-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -21,7 +23,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -40,7 +49,8 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" - - name: Docker build and push (nginx) + + - name: Docker build, sign and push (nginx) run: | IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:git-${GITHUB_SHA}" @@ -56,13 +66,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/miner_runner_cd_staging.yml b/.github/workflows/miner_runner_cd_staging.yml index 0073e481c..b80a2d6c6 100644 --- a/.github/workflows/miner_runner_cd_staging.yml +++ b/.github/workflows/miner_runner_cd_staging.yml @@ -13,6 +13,8 @@ env: MINER_IMAGE_REPO: "compute-horde-miner-staging" MINER_NGINX_IMAGE_REPO: "compute-horde-miner-nginx-staging" DIST_VERSION_PREFIX: "miner-runner-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -21,7 +23,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -40,7 +49,8 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" - - name: Docker build and push (nginx) + + - name: Docker build, sign and push (nginx) run: | IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_NGINX_REPO_NAME}:git-${GITHUB_SHA}" @@ -56,13 +66,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_cd_preprod.yml b/.github/workflows/validator_cd_preprod.yml index b0fce07b5..ec075de66 100644 --- a/.github/workflows/validator_cd_preprod.yml +++ b/.github/workflows/validator_cd_preprod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-preprod" DIST_VERSION_PREFIX: "validator-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_cd_prod.yml b/.github/workflows/validator_cd_prod.yml index f2e366b9f..8995a5805 100644 --- a/.github/workflows/validator_cd_prod.yml +++ b/.github/workflows/validator_cd_prod.yml @@ -10,6 +10,8 @@ env: STAGING_DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-staging" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator" DIST_VERSION_PREFIX: "validator-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Promote image run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${STAGING_DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -29,13 +38,16 @@ jobs: echo "${{ secrets.DOCKERHUB_KEY }}" | docker login -u "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin docker push "${IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_cd_staging.yml b/.github/workflows/validator_cd_staging.yml index 70eb361fe..e1aa73282 100644 --- a/.github/workflows/validator_cd_staging.yml +++ b/.github/workflows/validator_cd_staging.yml @@ -10,6 +10,8 @@ env: TAG_VERSION: "v0-latest" DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-staging" DIST_VERSION_PREFIX: "validator-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -18,7 +20,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -36,13 +45,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_ci.yml b/.github/workflows/validator_ci.yml index a4213a767..3fc63c25d 100644 --- a/.github/workflows/validator_ci.yml +++ b/.github/workflows/validator_ci.yml @@ -2,9 +2,9 @@ name: Run Validator linter and tests on: push: - branches: [master, main] + branches: [ master, main ] pull_request: - branches: [master, main] + branches: [ master, main ] env: PYTHON_DEFAULT_VERSION: "3.11" @@ -19,15 +19,19 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Run linters run: nox -vs lint + test: timeout-minutes: 10 runs-on: ubuntu-latest @@ -38,13 +42,16 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" + - name: Install dependencies run: python -m pip install --upgrade nox 'pdm>=2.12,<3' + - name: Setup common virtualenv # In order not to exhaust disk on GitHub runner, we use one single # virtualenv for all pdm projects: miner, executor, validator. @@ -54,14 +61,19 @@ jobs: echo "$.venv/bin" >> $GITHUB_PATH echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> $GITHUB_ENV echo "PDM_IGNORE_SAVED_PYTHON=1" >> $GITHUB_ENV + - name: Prepare environment run: ./setup-dev.sh + - name: Run dockerized services run: docker compose up -d --wait + - name: Run migrations run: cd app/src && pdm run python manage.py wait_for_database --timeout 120 && pdm run python manage.py migrate + - name: Run unit tests run: pdm run nox -vs test --no-venv + - name: Stop dockerized services if: success() || failure() run: docker compose down -v diff --git a/.github/workflows/validator_runner_cd_preprod.yml b/.github/workflows/validator_runner_cd_preprod.yml index 43fdc26d2..0eda741c0 100644 --- a/.github/workflows/validator_runner_cd_preprod.yml +++ b/.github/workflows/validator_runner_cd_preprod.yml @@ -11,6 +11,8 @@ env: DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-runner-preprod" VALIDATOR_IMAGE_REPO: "compute-horde-validator-preprod" DIST_VERSION_PREFIX: "validator-runner-preprod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -19,7 +21,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -37,13 +46,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_runner_cd_prod.yml b/.github/workflows/validator_runner_cd_prod.yml index 7dbedfd1e..0a628d628 100644 --- a/.github/workflows/validator_runner_cd_prod.yml +++ b/.github/workflows/validator_runner_cd_prod.yml @@ -11,6 +11,8 @@ env: DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-runner" VALIDATOR_IMAGE_REPO: "compute-horde-validator" DIST_VERSION_PREFIX: "validator-runner-prod" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -19,10 +21,17 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" - SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" + SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}"\ cd "${BUILD_DIRECTORY}" && \ docker build \ @@ -37,13 +46,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/.github/workflows/validator_runner_cd_staging.yml b/.github/workflows/validator_runner_cd_staging.yml index 6b05fb219..84434ad97 100644 --- a/.github/workflows/validator_runner_cd_staging.yml +++ b/.github/workflows/validator_runner_cd_staging.yml @@ -11,6 +11,8 @@ env: DOCKER_REPO_NAME: "backenddevelopersltd/compute-horde-validator-runner-staging" VALIDATOR_IMAGE_REPO: "compute-horde-validator-staging" DIST_VERSION_PREFIX: "validator-runner-staging" + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.DOCKERHUB_DELEGATED_KEY_PASS }} jobs: deploy: @@ -19,7 +21,14 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker build and push + + - name: Install DCT key + uses: ./.github/actions/install-dct-key + with: + key-id: "${{ secrets.DOCKERHUB_DELEGATED_KEY_ID }}" + key-content: "${{ secrets.DOCKERHUB_DELEGATED_KEY }}" + + - name: Docker build, sign and push run: | IMAGE_NAME="${DOCKER_REPO_NAME}:${TAG_VERSION}" SHA_IMAGE_NAME="${DOCKER_REPO_NAME}:git-${GITHUB_SHA}" @@ -37,13 +46,16 @@ jobs: docker push "${IMAGE_NAME}" docker push "${SHA_IMAGE_NAME}" + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Set Dist Version run: | BUILD_NUMBER="${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" echo "DIST_VERSION=${DIST_VERSION_PREFIX}-${{ steps.date.outputs.date }}-${BUILD_NUMBER}" >> $GITHUB_ENV + - name: Create Tag uses: actions/github-script@v6 with: diff --git a/dct_pull/Dockerfile b/dct_pull/Dockerfile new file mode 100644 index 000000000..788435270 --- /dev/null +++ b/dct_pull/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest +RUN apk add --no-cache git bash + +COPY dct_pull.sh / +RUN chmod +x /dct_pull.sh + +ENTRYPOINT "/dct_pull.sh" diff --git a/dct_pull/README.md b/dct_pull/README.md new file mode 100644 index 000000000..ecaeabfa4 --- /dev/null +++ b/dct_pull/README.md @@ -0,0 +1,12 @@ +# DCT pull +Gets around the issue of watchtower not respecting DCT. + +This is a small docker image that: +- uses passed-in docker socket +- wakes up once per minute +- with the DCT signature checks enabled, tries to pull in new image versions + +That's it. + +There is an assumption that there is also an instance of watchtower running - it will update +running containers to use the pulled images. \ No newline at end of file diff --git a/dct_pull/dct_pull.sh b/dct_pull/dct_pull.sh new file mode 100755 index 000000000..6420dfe8a --- /dev/null +++ b/dct_pull/dct_pull.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +export DOCKER_CONTENT_TRUST=1 + +if [ $# -eq 0 ]; then + echo "Provide docker image names as arguments" + exit 1 +fi + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +while true; do + for image in "$@"; do + echo -e "\n\nPulling $image" + docker pull $image && echo -e "${GREEN}Successfully pulled signed image: $image${NC}" || echo -e "${RED}Failed to pull signed image: $image${NC}" + done + sleep 1m +done + diff --git a/dct_pull/publish-image.sh b/dct_pull/publish-image.sh new file mode 100755 index 000000000..2120782db --- /dev/null +++ b/dct_pull/publish-image.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -eux -o pipefail + +export DOCKER_CONTENT_TRUST=1 + +IMAGE_NAME=backenddevelopersltd/compute-horde-dct-pull:v0-latest + +docker build \ + --platform=linux/amd64 \ + -t $IMAGE_NAME \ + . + +if [ -z "$(docker info 2>/dev/null | grep 'Username' | awk '{print $2}')" ]; then + echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin +fi + +docker push "$IMAGE_NAME" diff --git a/executor/app/publish-image.sh b/executor/app/publish-image.sh index d431ea9dc..7ab7f89f3 100755 --- a/executor/app/publish-image.sh +++ b/executor/app/publish-image.sh @@ -3,4 +3,5 @@ set -eux -o pipefail source ./build-image.sh echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/executor/app/src/compute_horde_executor/settings.py b/executor/app/src/compute_horde_executor/settings.py index 70eea008e..f0fd41ba4 100644 --- a/executor/app/src/compute_horde_executor/settings.py +++ b/executor/app/src/compute_horde_executor/settings.py @@ -43,6 +43,7 @@ def wrapped(*args, **kwargs): ENV = env("ENV", default="prod") + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = env("SECRET_KEY", default="dummy") diff --git a/executor/deploy.sh b/executor/deploy.sh index a3f5229c6..a3568cefd 100755 --- a/executor/deploy.sh +++ b/executor/deploy.sh @@ -22,6 +22,10 @@ SERVICES=$(docker-compose ps --services 2>&1 > /dev/stderr \ # shellcheck disable=2086 docker-compose stop $SERVICES +# explicitly pull the docker compose images to verify DCT +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + # start the app container only in order to perform migrations docker-compose up -d db # in case it hasn't been launched before docker-compose run --rm app sh -c "python manage.py wait_for_database --timeout 10; python manage.py migrate" diff --git a/install_miner.sh b/install_miner.sh index a9b01eb3a..b8d153072 100755 --- a/install_miner.sh +++ b/install_miner.sh @@ -106,8 +106,8 @@ sudo usermod -aG docker $USER # install cuda sudo apt-get install -y linux-headers-$(uname -r) DISTRIBUTION=$(. /etc/os-release; echo $ID$VERSION_ID | sed -e 's/\.//g') -wget "https://developer.download.nvidia.com/compute/cuda/repos/$DISTRIBUTION/x86_64/cuda-keyring_1.0-1_all.deb" -sudo dpkg -i cuda-keyring_1.0-1_all.deb +wget "https://developer.download.nvidia.com/compute/cuda/repos/$DISTRIBUTION/x86_64/cuda-keyring_1.1-1_all.deb" +sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt-get update sudo apt-get -y install cuda-drivers @@ -128,7 +128,7 @@ ENDSSH # start a new ssh connection so that usermod changes are effective ssh "$SSH_DESTINATION" <<'ENDSSH' set -euxo pipefail -mkdir ~/compute_horde_miner +mkdir -p ~/compute_horde_miner cd ~/compute_horde_miner cat > docker-compose.yml <<'ENDDOCKERCOMPOSE' @@ -151,9 +151,13 @@ services: restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 60 --cleanup --label-enable + command: --interval 60 --cleanup --label-enable --no-pull ENDDOCKERCOMPOSE +# Pull images, verifying they are signed +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + cat > .env < docker-compose.yml <<'ENDDOCKERCOMPOSE' @@ -97,9 +97,13 @@ services: restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 60 --cleanup --label-enable + command: --interval 60 --cleanup --label-enable --no-pull ENDDOCKERCOMPOSE +# Pull images, verifying they are signed +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + cat > .env <&1 > /dev/stderr \ # shellcheck disable=2086 docker-compose stop $SERVICES +# explicitly pull the docker compose images to verify DCT +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + # start the app container only in order to perform migrations docker-compose up -d db # in case it hasn't been launched before docker-compose run --rm app sh -c "python manage.py wait_for_database --timeout 10; python manage.py migrate" diff --git a/miner/envs/runner/README.md b/miner/envs/runner/README.md index 5cf9b1edc..b4d01c30f 100644 --- a/miner/envs/runner/README.md +++ b/miner/envs/runner/README.md @@ -52,7 +52,7 @@ services: restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 60 --cleanup --label-enable + command: --interval 60 --cleanup --label-enable --no-pull ``` diff --git a/miner/envs/runner/_publish-image.sh b/miner/envs/runner/_publish-image.sh index f3ae8143a..cefc183e6 100755 --- a/miner/envs/runner/_publish-image.sh +++ b/miner/envs/runner/_publish-image.sh @@ -5,4 +5,5 @@ if [ -z "$(docker info 2>/dev/null | grep 'Username' | awk '{print $2}')" ]; the echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin fi +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/miner/envs/runner/data/docker-compose.yml b/miner/envs/runner/data/docker-compose.yml index dd1d963f0..dc870c6e5 100644 --- a/miner/envs/runner/data/docker-compose.yml +++ b/miner/envs/runner/data/docker-compose.yml @@ -144,6 +144,18 @@ services: logging: <<: *logging + dct-pull: + image: backenddevelopersltd/compute-horde-dct-pull:v0-latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: backenddevelopersltd/compute-horde-dct-pull:v0-latest backenddevelopersltd/${MINER_IMAGE_REPO}:${MINER_IMAGE_TAG:-v0-latest} backenddevelopersltd/${MINER_NGINX_IMAGE_REPO}:${MINER_IMAGE_TAG:-v0-latest} + logging: + <<: *logging + labels: + - "com.centurylinklabs.watchtower.enable=true" + + volumes: redis: db: diff --git a/miner/envs/runner/entrypoint.sh b/miner/envs/runner/entrypoint.sh index 0e87242f2..1910974e1 100644 --- a/miner/envs/runner/entrypoint.sh +++ b/miner/envs/runner/entrypoint.sh @@ -1,6 +1,10 @@ #!/bin/sh set -eu +# explicitly pull the docker compose images to verify DCT +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + docker compose up --detach --wait --force-recreate while true diff --git a/miner/envs/runner/nginx/publish-prod.sh b/miner/envs/runner/nginx/publish-prod.sh index 910abd02b..aea343dc7 100755 --- a/miner/envs/runner/nginx/publish-prod.sh +++ b/miner/envs/runner/nginx/publish-prod.sh @@ -3,4 +3,5 @@ set -eux -o pipefail source build-prod.sh echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/miner/envs/runner/nginx/publish-staging.sh b/miner/envs/runner/nginx/publish-staging.sh index d45b3ffb2..46d56ef32 100755 --- a/miner/envs/runner/nginx/publish-staging.sh +++ b/miner/envs/runner/nginx/publish-staging.sh @@ -3,4 +3,5 @@ set -eux -o pipefail source build-staging.sh echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/validator/app/envs/job/publish-image.sh b/validator/app/envs/job/publish-image.sh index b0f0ae1af..156edefcd 100644 --- a/validator/app/envs/job/publish-image.sh +++ b/validator/app/envs/job/publish-image.sh @@ -3,5 +3,6 @@ set -eux -o pipefail source ./build-image.sh echo "$GITHUB_CR_PAT" | docker login ghcr.io -u USERNAME --password-stdin +export DOCKER_CONTENT_TRUST=1 docker push "$BASE_IMAGE_NAME" docker push "$IMAGE_NAME" diff --git a/validator/app/publish-image.sh b/validator/app/publish-image.sh index d431ea9dc..7ab7f89f3 100755 --- a/validator/app/publish-image.sh +++ b/validator/app/publish-image.sh @@ -3,4 +3,5 @@ set -eux -o pipefail source ./build-image.sh echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/validator/app/src/compute_horde_validator/settings.py b/validator/app/src/compute_horde_validator/settings.py index 83536a1f6..b67461cff 100644 --- a/validator/app/src/compute_horde_validator/settings.py +++ b/validator/app/src/compute_horde_validator/settings.py @@ -43,6 +43,7 @@ def wrapped(*args, **kwargs): ENV = env("ENV", default="prod") + DEFAULT_ADMIN_PASSWORD = env("DEFAULT_ADMIN_PASSWORD", default=None) DEFAULT_ADMIN_USERNAME = env("DEFAULT_ADMIN_USERNAME", default="admin") DEFAULT_ADMIN_EMAIL = env("DEFAULT_ADMIN_EMAIL", default="admin@admin.com") diff --git a/validator/envs/runner/README.md b/validator/envs/runner/README.md index c9e6d057d..9855beaa4 100644 --- a/validator/envs/runner/README.md +++ b/validator/envs/runner/README.md @@ -35,7 +35,7 @@ services: restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 60 --cleanup --label-enable + command: --interval 60 --cleanup --label-enable --no-pull ``` diff --git a/validator/envs/runner/_publish-image.sh b/validator/envs/runner/_publish-image.sh index f3ae8143a..cefc183e6 100755 --- a/validator/envs/runner/_publish-image.sh +++ b/validator/envs/runner/_publish-image.sh @@ -5,4 +5,5 @@ if [ -z "$(docker info 2>/dev/null | grep 'Username' | awk '{print $2}')" ]; the echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin fi +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME" diff --git a/validator/envs/runner/data/docker-compose.yml b/validator/envs/runner/data/docker-compose.yml index dc4593ff4..d089ede10 100644 --- a/validator/envs/runner/data/docker-compose.yml +++ b/validator/envs/runner/data/docker-compose.yml @@ -129,6 +129,17 @@ services: labels: - "com.centurylinklabs.watchtower.enable=true" + dct-pull: + image: backenddevelopersltd/compute-horde-dct-pull:v0-latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: backenddevelopersltd/compute-horde-dct-pull:v0-latest backenddevelopersltd/${VALIDATOR_IMAGE_REPO}:v0-latest backenddevelopersltd/compute-horde-validator-nginx:v0-latest + logging: + <<: *logging + labels: + - "com.centurylinklabs.watchtower.enable=true" + volumes: redis: diff --git a/validator/envs/runner/entrypoint.sh b/validator/envs/runner/entrypoint.sh index 0e87242f2..1910974e1 100644 --- a/validator/envs/runner/entrypoint.sh +++ b/validator/envs/runner/entrypoint.sh @@ -1,6 +1,10 @@ #!/bin/sh set -eu +# explicitly pull the docker compose images to verify DCT +export DOCKER_CONTENT_TRUST=1 +docker compose convert --images | sort -u | xargs -n 1 docker pull + docker compose up --detach --wait --force-recreate while true diff --git a/validator/envs/runner/nginx/publish-image.sh b/validator/envs/runner/nginx/publish-image.sh index b78b8c2dd..a5b4975be 100755 --- a/validator/envs/runner/nginx/publish-image.sh +++ b/validator/envs/runner/nginx/publish-image.sh @@ -5,4 +5,5 @@ source ./build-image.sh if [ -z "$(docker info 2>/dev/null | grep 'Username' | awk '{print $2}')" ]; then echo "$DOCKERHUB_PAT" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin fi +export DOCKER_CONTENT_TRUST=1 docker push "$IMAGE_NAME"