Update CI files

[noissue]

.ci/scripts/check_release.py

@@ -29,7 +29,7 @@ def main():
         "--branches",
         default="supported",
         help="A comma separated list of branches to check for releases. Can also use keyword: "
-        "'supported'. Defaults to 'supported', see `ci_update_branches` in "
+        "'supported'. Defaults to 'supported', see `supported_release_branches` in "
         "`plugin_template.yml`.",
     )
     opts = parser.parse_args()
@@ -46,12 +46,15 @@ def main():
         if branches == "supported":
             with open(f"{d}/template_config.yml", mode="r") as f:
                 tc = yaml.safe_load(f)
-                branches = tc["ci_update_branches"]
-            branches.append(DEFAULT_BRANCH)
+                branches = set(tc["supported_release_branches"])
+            latest_release_branch = tc["latest_release_branch"]
+            if latest_release_branch is not None:
+                branches.add(latest_release_branch)
+            branches.add(DEFAULT_BRANCH)
         else:
-            branches = branches.split(",")
+            branches = set(branches.split(","))
 
-        if diff := set(branches) - set(available_branches):
+        if diff := branches - set(available_branches):
             print(f"Supplied branches contains non-existent branches! {diff}")
             exit(1)
 

.ci/scripts/collect_changes.py

@@ -19,9 +19,11 @@
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
     "start_string",
-    "<!-- towncrier release notes start -->\n"
-    if CHANGELOG_FILE.endswith(".md")
-    else ".. towncrier release notes start\n",
+    (
+        "<!-- towncrier release notes start -->\n"
+        if CHANGELOG_FILE.endswith(".md")
+        else ".. towncrier release notes start\n"
+    ),
 )
 TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
 

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-301-g83f0607
+2021.08.26-310-g59cd732

.github/workflows/create-branch.yml

@@ -71,6 +71,19 @@ jobs:
         run: |
           find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
 
+      - name: Checkout plugin template
+        uses: actions/checkout@v3
+        with:
+          repository: pulp/plugin_template
+          path: plugin_template
+          fetch-depth: 0
+
+      - name: Update CI branches in template_config
+        working-directory: plugin_template
+        run: |
+          python3 ./plugin-template pulp_file --github --latest-release-branch "${NEW_BRANCH}"
+          git add -A
+
       - name: Make a PR with version bump and without CHANGES/*
         uses: peter-evans/create-pull-request@v4
         with:

.github/workflows/publish.yml

@@ -0,0 +1,322 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_file' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+---
+name: "File Publish Release"
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+"
+
+defaults:
+  run:
+    working-directory: "pulp_file"
+
+jobs:
+  build:
+    uses: "./.github/workflows/build.yml"
+
+  build-bindings-docs:
+    needs:
+      - "build"
+    runs-on: "ubuntu-latest"
+    # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests
+    env:
+      TEST: "docs"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_file"
+
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          repository: "pulp/pulp-openapi-generator"
+          path: "pulp-openapi-generator"
+
+      - uses: "actions/setup-python@v4"
+        with:
+          python-version: "3.8"
+
+      - uses: "actions/download-artifact@v3"
+        with:
+          name: "plugin_package"
+          path: "pulp_file/dist/"
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "2.6"
+
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs
+          echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_file/.ci/assets/httpie/" >> $GITHUB_ENV
+          echo ::endgroup::
+
+      # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to
+      # setup the Pulp instance.
+      - name: "Before Install"
+        run: |
+          .github/workflows/scripts/before_install.sh
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
+
+      - name: "Install"
+        run: |
+          .github/workflows/scripts/install.sh
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
+
+      - name: "Install Python client"
+        run: |
+          .github/workflows/scripts/install_python_client.sh
+        shell: "bash"
+      - name: "Install Ruby client"
+        run: |
+          .github/workflows/scripts/install_ruby_client.sh
+        shell: "bash"
+      - name: "Upload python client packages"
+        uses: "actions/upload-artifact@v3"
+        with:
+          name: "python-client.tar"
+          path: "pulp_file/file-python-client.tar"
+          if-no-files-found: "error"
+
+      - name: "Upload python client docs"
+        uses: "actions/upload-artifact@v3"
+        with:
+          name: "python-client-docs.tar"
+          path: "pulp_file/file-python-client-docs.tar"
+          if-no-files-found: "error"
+      - name: "Upload ruby client packages"
+        uses: "actions/upload-artifact@v3"
+        with:
+          name: "ruby-client.tar"
+          path: "pulp_file/file-ruby-client.tar"
+          if-no-files-found: "error"
+      - name: Build docs
+        run: |
+          export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
+          export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py
+          make -C docs/ PULP_URL="https://pulp" diagrams html
+          tar -cvf docs/docs.tar docs/_build
+
+      - name: "Upload built docs"
+        uses: actions/upload-artifact@v3
+        with:
+          name: "docs.tar"
+          path: "pulp_file/docs/docs.tar"
+
+      - name: "Logs"
+        if: always()
+        run: |
+          echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
+          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
+          docker images || true
+          docker ps -a || true
+          docker logs pulp || true
+          docker exec pulp ls -latr /etc/yum.repos.d/ || true
+          docker exec pulp cat /etc/yum.repos.d/* || true
+          docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
+  publish-package:
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-bindings-docs"
+
+    env:
+      GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_file"
+
+      - uses: "actions/download-artifact@v3"
+        with:
+          name: "plugin_package"
+          path: "pulp_file/dist/"
+
+      - uses: "actions/setup-python@v4"
+        with:
+          python-version: "3.8"
+
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install twine
+          echo ::endgroup::
+
+      - name: "Setting secrets"
+        run: |
+          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
+
+      - name: "Deploy plugin to pypi"
+        run: |
+          .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }}
+  publish-python-bindings:
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-bindings-docs"
+
+    env:
+      GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_file"
+
+      - name: "Download Python client"
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "python-client.tar"
+          path: "pulp_file/"
+
+      - name: "Untar python client packages"
+        run: |
+          tar -xvf file-python-client.tar
+
+      - uses: "actions/setup-python@v4"
+        with:
+          python-version: "3.8"
+
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install twine
+          echo ::endgroup::
+
+      - name: "Setting secrets"
+        run: |
+          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
+
+      - name: "Publish client to pypi"
+        run: |
+          bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }}
+  publish-ruby-bindings:
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-bindings-docs"
+
+    env:
+      GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_file"
+
+      - name: "Download Ruby client"
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "ruby-client.tar"
+          path: "pulp_file/"
+
+      - name: "Untar Ruby client packages"
+        run: |
+          tar -xvf file-ruby-client.tar
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "2.6"
+
+      - name: "Setting secrets"
+        run: |
+          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
+
+      - name: "Publish client to rubygems"
+        run: |
+          bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }}
+  publish-docs:
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-bindings-docs"
+
+    env:
+      GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_file"
+
+      - uses: "actions/setup-python@v4"
+        with:
+          python-version: "3.8"
+
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install 'packaging~=21.3' requests
+          echo ::endgroup::
+
+      - name: "Setting secrets"
+        run: |
+          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
+
+      - name: "Download built docs"
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "docs.tar"
+          path: "pulp_file/"
+
+      - name: "Download Python client docs"
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "python-client-docs.tar"
+          path: "pulp_file/"
+
+      - name: "Publish docs to pulpproject.org"
+        run: |
+          tar -xvf docs.tar
+          .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }}
+
+  create-gh-release:
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-bindings-docs"
+      - "publish-package"
+      - "publish-python-bindings"
+      - "publish-ruby-bindings"
+      - "publish-docs"
+
+    steps:
+      - name: "Create release on GitHub"
+        uses: "actions/github-script@v7"
+        env:
+          TAG_NAME: "${{ github.ref_name }}"
+        with:
+          script: |
+            const { TAG_NAME } = process.env;
+
+            await github.rest.repos.createRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: TAG_NAME,
+              make_latest: "legacy",
+            });