From a01ce157d5ade48b346d4e73f741b82dbaf99f63 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:39:52 +0200 Subject: [PATCH 01/14] ci: Optimize CI workflow to run tests and linting only on relevant file changes --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05544648e..2e0cf5300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,12 +2,17 @@ name: Test suite on: push: + paths: + - 'backend/**' + - 'frontend/**' jobs: backend-test: name: Test Backend runs-on: ubuntu-latest + # run only when files have changed in the backend directory + if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'backend/') }} steps: - uses: actions/checkout@v3 - name: Run Backend Tests @@ -51,6 +56,8 @@ jobs: backend-lint: name: Lint Backend runs-on: ubuntu-latest + # run only when files have changed in the backend directory + if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'backend/') }} steps: - uses: actions/checkout@v3 - name: Lint Backend @@ -60,6 +67,8 @@ jobs: frontend-test: name: Test Frontend runs-on: ubuntu-latest + # run only when files have changed in the frontend directory + if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'frontend/') }} steps: - uses: actions/checkout@v3 - name: Run Frontend Tests @@ -104,6 +113,8 @@ jobs: frontend-lint: name: Lint Frontend runs-on: ubuntu-latest + # run only when files have changed in the frontend directory + if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'frontend/') }} steps: - uses: actions/checkout@v3 - run: sudo docker-compose --env-file .env-github-actions run client yarn lint From 52770f13e8ba5e1ba87cf8dbf3cd6c752b05d8ab Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:42:03 +0200 Subject: [PATCH 02/14] refactor: Only run storybook workflow on relevant changes --- .github/workflows/storybook.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml index 5aa502e97..0b25c0465 100644 --- a/.github/workflows/storybook.yml +++ b/.github/workflows/storybook.yml @@ -5,6 +5,11 @@ on: branches: - main - develop + paths: + - 'frontend/**' + - '.github/workflows/storybook.yml' + - '.storybook/**' + - '.yarn' pull_request: permissions: From d46b175d95e60bcf89bfda8033bfa7a20072e83d Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:44:44 +0200 Subject: [PATCH 03/14] temp: Update frontend file to trigger test --- frontend/src/components/App/App.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/App/App.tsx b/frontend/src/components/App/App.tsx index afdda65bc..1796b5eb4 100644 --- a/frontend/src/components/App/App.tsx +++ b/frontend/src/components/App/App.tsx @@ -55,6 +55,8 @@ const App = () => { return

Error: {error}

; } + // test comment to trigger tests + return ( <> From 7cf758f3b89e6a04adec53fc245933d9eaff4c83 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:47:12 +0200 Subject: [PATCH 04/14] temp: Remove workflow paths condition --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e0cf5300..ad214f4f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,6 @@ name: Test suite on: push: - paths: - - 'backend/**' - - 'frontend/**' jobs: From db2fd0a6c94791517396799af4c31d350ba71c06 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:49:42 +0200 Subject: [PATCH 05/14] refactor: Split CI workflow for frontend and backend test & linting --- .github/workflows/ci-backend.yml | 60 +++++++++++++++ .github/workflows/ci-frontend.yml | 59 +++++++++++++++ .github/workflows/ci.yml | 117 ------------------------------ 3 files changed, 119 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/ci-backend.yml create mode 100644 .github/workflows/ci-frontend.yml delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml new file mode 100644 index 000000000..2e6156413 --- /dev/null +++ b/.github/workflows/ci-backend.yml @@ -0,0 +1,60 @@ +name: Backend Test & Lint + +on: + push: + paths: + - 'backend/**' + +jobs: + + backend-test: + name: Test Backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run Backend Tests + run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage run manage.py test" + - name: Generate Backend Coverage Report (Inline) + run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage report --show-missing" + + # Generate coverage badge (only for main and develop branches) + - name: Generate Backend Coverage Report (XML) and Badge + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' + run: | + sudo docker-compose --env-file .env-github-actions run server bash -c "coverage xml" + sudo docker-compose --env-file .env-github-actions run server bash -c "genbadge coverage -i coverage.xml -o coverage-backend-badge-new.svg -n \"Backend Code Coverage\"" + # Push coverage badge to separate branch (only for main and develop branches) + - name: Push Backend Coverage Badge to separate branch + continue-on-error: true + if: github.ref == 'refs/heads/develop' + run: | + if git ls-remote --heads origin code-coverage-badges; then + git fetch origin code-coverage-badges + git checkout code-coverage-badges + else + git checkout -b code-coverage-badges + git push origin code-coverage-badges + fi + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + rm backend/coverage-backend-badge.svg -f + mv backend/coverage-backend-badge-new.svg backend/coverage-backend-badge.svg + git add backend/coverage-backend-badge.svg + git commit -m "Add backend coverage badge for commit $GITHUB_SHA" + git push origin code-coverage-badges + # Check if there are any changes + if git diff --staged --quiet; then + echo "No changes in coverage badge. Skipping commit and push." + else + git commit -m "Add backend coverage badge for commit $GITHUB_SHA" + git push origin code-coverage-badges + fi + + backend-lint: + name: Lint Backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Lint Backend + continue-on-error: false + run: sudo docker-compose --env-file .env-github-actions run server bash -c "flake8" diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml new file mode 100644 index 000000000..a72c7e2cf --- /dev/null +++ b/.github/workflows/ci-frontend.yml @@ -0,0 +1,59 @@ +name: Frontend Test & Lint + +on: + push: + paths: + - 'frontend/**' + +jobs: + + frontend-test: + name: Test Frontend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run Frontend Tests + run: sudo docker-compose --env-file .env-github-actions run client yarn test:ci + + frontend-coverage-badge: + name: Generate Frontend Coverage Badge + needs: frontend-test + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop' + steps: + - uses: actions/checkout@v3 + - name: Generate Frontend Coverage Report (XML) and Badge + run: | + sudo docker-compose --env-file .env-github-actions run client yarn test:ci + sudo docker-compose --env-file .env-github-actions run client yarn coverage-badges -s public/coverage/coverage-summary.json -o public/coverage/coverage-frontend-badge-new.svg --label 'Frontend Code Coverage' + - name: Push Frontend Coverage Badge to separate branch + continue-on-error: true + run: | + if git ls-remote --heads origin code-coverage-badges; then + git fetch origin code-coverage-badges + git checkout code-coverage-badges + else + git checkout -b code-coverage-badges + git push origin code-coverage-badges + fi + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + rm frontend/coverage-frontend-badge.svg -f + sudo mv frontend/public/coverage/coverage-frontend-badge-new.svg frontend/coverage-frontend-badge.svg + git add frontend/coverage-frontend-badge.svg + git commit -m "Add frontend coverage badge for commit $GITHUB_SHA" + git push origin code-coverage-badges + # Check if there are any changes + if git diff --staged --quiet; then + echo "No changes in coverage badge. Skipping commit and push." + else + git commit -m "Add frontend coverage badge for commit $GITHUB_SHA" + git push origin code-coverage-badges + fi + + frontend-lint: + name: Lint Frontend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: sudo docker-compose --env-file .env-github-actions run client yarn lint diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index ad214f4f1..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Test suite - -on: - push: - -jobs: - - backend-test: - name: Test Backend - runs-on: ubuntu-latest - # run only when files have changed in the backend directory - if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'backend/') }} - steps: - - uses: actions/checkout@v3 - - name: Run Backend Tests - run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage run manage.py test" - - name: Generate Backend Coverage Report (Inline) - run: sudo docker-compose --env-file .env-github-actions run server bash -c "coverage report --show-missing" - - # Generate coverage badge (only for main and develop branches) - - name: Generate Backend Coverage Report (XML) and Badge - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' - run: | - sudo docker-compose --env-file .env-github-actions run server bash -c "coverage xml" - sudo docker-compose --env-file .env-github-actions run server bash -c "genbadge coverage -i coverage.xml -o coverage-backend-badge-new.svg -n \"Backend Code Coverage\"" - # Push coverage badge to separate branch (only for main and develop branches) - - name: Push Backend Coverage Badge to separate branch - continue-on-error: true - if: github.ref == 'refs/heads/develop' - run: | - if git ls-remote --heads origin code-coverage-badges; then - git fetch origin code-coverage-badges - git checkout code-coverage-badges - else - git checkout -b code-coverage-badges - git push origin code-coverage-badges - fi - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - rm backend/coverage-backend-badge.svg -f - mv backend/coverage-backend-badge-new.svg backend/coverage-backend-badge.svg - git add backend/coverage-backend-badge.svg - git commit -m "Add backend coverage badge for commit $GITHUB_SHA" - git push origin code-coverage-badges - # Check if there are any changes - if git diff --staged --quiet; then - echo "No changes in coverage badge. Skipping commit and push." - else - git commit -m "Add backend coverage badge for commit $GITHUB_SHA" - git push origin code-coverage-badges - fi - - backend-lint: - name: Lint Backend - runs-on: ubuntu-latest - # run only when files have changed in the backend directory - if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'backend/') }} - steps: - - uses: actions/checkout@v3 - - name: Lint Backend - continue-on-error: false - run: sudo docker-compose --env-file .env-github-actions run server bash -c "flake8" - - frontend-test: - name: Test Frontend - runs-on: ubuntu-latest - # run only when files have changed in the frontend directory - if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'frontend/') }} - steps: - - uses: actions/checkout@v3 - - name: Run Frontend Tests - run: sudo docker-compose --env-file .env-github-actions run client yarn test:ci - - frontend-coverage-badge: - name: Generate Frontend Coverage Badge - needs: frontend-test - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/develop' - steps: - - uses: actions/checkout@v3 - - name: Generate Frontend Coverage Report (XML) and Badge - run: | - sudo docker-compose --env-file .env-github-actions run client yarn test:ci - sudo docker-compose --env-file .env-github-actions run client yarn coverage-badges -s public/coverage/coverage-summary.json -o public/coverage/coverage-frontend-badge-new.svg --label 'Frontend Code Coverage' - - name: Push Frontend Coverage Badge to separate branch - continue-on-error: true - run: | - if git ls-remote --heads origin code-coverage-badges; then - git fetch origin code-coverage-badges - git checkout code-coverage-badges - else - git checkout -b code-coverage-badges - git push origin code-coverage-badges - fi - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - rm frontend/coverage-frontend-badge.svg -f - sudo mv frontend/public/coverage/coverage-frontend-badge-new.svg frontend/coverage-frontend-badge.svg - git add frontend/coverage-frontend-badge.svg - git commit -m "Add frontend coverage badge for commit $GITHUB_SHA" - git push origin code-coverage-badges - # Check if there are any changes - if git diff --staged --quiet; then - echo "No changes in coverage badge. Skipping commit and push." - else - git commit -m "Add frontend coverage badge for commit $GITHUB_SHA" - git push origin code-coverage-badges - fi - - frontend-lint: - name: Lint Frontend - runs-on: ubuntu-latest - # run only when files have changed in the frontend directory - if: ${{ github.event_name == 'push' && contains(github.event.head_commit.modified, 'frontend/') }} - steps: - - uses: actions/checkout@v3 - - run: sudo docker-compose --env-file .env-github-actions run client yarn lint From 7905a6204a8e709b08e5d63c4659d6092bfb43bb Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:55:19 +0200 Subject: [PATCH 06/14] chore: Update CI workflows to include pull requests for backend and frontend --- .github/workflows/ci-backend.yml | 3 +++ .github/workflows/ci-frontend.yml | 3 +++ .github/workflows/storybook.yml | 6 +----- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 2e6156413..372e0e98a 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -4,6 +4,9 @@ on: push: paths: - 'backend/**' + pull_request: + paths: + - 'backend/**' jobs: diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index a72c7e2cf..1e6ea0c8a 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -4,6 +4,9 @@ on: push: paths: - 'frontend/**' + pull_request: + paths: + - 'frontend/**' jobs: diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml index 0b25c0465..f08990f16 100644 --- a/.github/workflows/storybook.yml +++ b/.github/workflows/storybook.yml @@ -6,11 +6,7 @@ on: - main - develop paths: - - 'frontend/**' - - '.github/workflows/storybook.yml' - - '.storybook/**' - - '.yarn' - pull_request: + - '!backend/**' permissions: contents: read From 6cb9ae1f0f55e4ee67a092aaa56430baeeee0bfb Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:56:40 +0200 Subject: [PATCH 07/14] chore: Remove backend path condition from CI workflow --- .github/workflows/storybook.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml index f08990f16..556caeff3 100644 --- a/.github/workflows/storybook.yml +++ b/.github/workflows/storybook.yml @@ -5,8 +5,6 @@ on: branches: - main - develop - paths: - - '!backend/**' permissions: contents: read From ad2f278e2fd89d8190dfaea2c889c9e03c546bdc Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 11:58:13 +0200 Subject: [PATCH 08/14] chore: Remove change from frontend & add one to backend to test trigger backend test & lint workflow --- backend/aml/base_settings.py | 2 ++ frontend/src/components/App/App.tsx | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/aml/base_settings.py b/backend/aml/base_settings.py index 3b0854ed6..9e7d02e34 100644 --- a/backend/aml/base_settings.py +++ b/backend/aml/base_settings.py @@ -39,6 +39,8 @@ # Application definition +# Trigger backend test & lint jobs + INSTALLED_APPS = [ 'admin_interface', 'django.contrib.admin', diff --git a/frontend/src/components/App/App.tsx b/frontend/src/components/App/App.tsx index 1796b5eb4..afdda65bc 100644 --- a/frontend/src/components/App/App.tsx +++ b/frontend/src/components/App/App.tsx @@ -55,8 +55,6 @@ const App = () => { return

Error: {error}

; } - // test comment to trigger tests - return ( <> From 4c5cd078256616e9b562d62c7fce0cff23b875a4 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 12:00:16 +0200 Subject: [PATCH 09/14] refactor: Remove trigger backend test & lint jobs from base settings --- backend/aml/base_settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/aml/base_settings.py b/backend/aml/base_settings.py index 9e7d02e34..3b0854ed6 100644 --- a/backend/aml/base_settings.py +++ b/backend/aml/base_settings.py @@ -39,8 +39,6 @@ # Application definition -# Trigger backend test & lint jobs - INSTALLED_APPS = [ 'admin_interface', 'django.contrib.admin', From 339e2f480a4b8744127e092959c172180663be1f Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 12:00:40 +0200 Subject: [PATCH 10/14] ci: Start storybook workflow only when specific files have changes in a push or PR --- .github/workflows/storybook.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml index 556caeff3..1a5a42935 100644 --- a/.github/workflows/storybook.yml +++ b/.github/workflows/storybook.yml @@ -5,6 +5,17 @@ on: branches: - main - develop + paths: + - 'frontend/**' + - '.github/workflows/storybook.yml' + - '.yarn/**' + - '.storybook/**' + pull_request: + paths: + - 'frontend/**' + - '.github/workflows/storybook.yml' + - '.yarn/**' + - '.storybook/**' permissions: contents: read From b0e48b925a394b6ba61f554256a3a3c7a7210ef2 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 12:02:17 +0200 Subject: [PATCH 11/14] chore: Remove trailing whitespace in App.tsx file --- frontend/src/components/App/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/App/App.tsx b/frontend/src/components/App/App.tsx index afdda65bc..2ec53c2ad 100644 --- a/frontend/src/components/App/App.tsx +++ b/frontend/src/components/App/App.tsx @@ -77,7 +77,7 @@ const App = () => { - + {/* Experiment Collection */} From 6e23e33864f32b9a5dea209f9c55fa7592174737 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 12:03:01 +0200 Subject: [PATCH 12/14] chore: Remove trailing whitespace in App.tsx file --- frontend/src/components/App/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/App/App.tsx b/frontend/src/components/App/App.tsx index 2ec53c2ad..afdda65bc 100644 --- a/frontend/src/components/App/App.tsx +++ b/frontend/src/components/App/App.tsx @@ -77,7 +77,7 @@ const App = () => { - + {/* Experiment Collection */} From 9ae176afa1b00502bf5ad681d365ddd04ffbda27 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Tue, 25 Jun 2024 12:08:08 +0200 Subject: [PATCH 13/14] chore: Update CI workflows to include .yarn folder in paths for frontend and pull requests --- .github/workflows/ci-frontend.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index 1e6ea0c8a..f12a6fabd 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -4,9 +4,11 @@ on: push: paths: - 'frontend/**' + - '.yarn' pull_request: paths: - 'frontend/**' + - '.yarn' jobs: From d759882bbb66ed59842b426f791be3f45aadfed3 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Mon, 1 Jul 2024 13:00:13 +0200 Subject: [PATCH 14/14] chore: Update CI workflows to include changes in their own workflow file, .env-github-actions and docker-compose.yaml as conditions to run --- .github/workflows/ci-backend.yml | 6 ++++++ .github/workflows/ci-frontend.yml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 372e0e98a..7e3f72892 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -4,9 +4,15 @@ on: push: paths: - 'backend/**' + - '.github/workflows/ci-backend.yml' + - 'docker-compose.yaml' + - '.env-github-actions' pull_request: paths: - 'backend/**' + - '.github/workflows/ci-backend.yml' + - 'docker-compose.yaml' + - '.env-github-actions' jobs: diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index f12a6fabd..6019cd9d8 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -5,10 +5,14 @@ on: paths: - 'frontend/**' - '.yarn' + - '.github/workflows/ci-frontend.yml' + - 'docker-compose.yaml' pull_request: paths: - 'frontend/**' - '.yarn' + - '.github/workflows/ci-frontend.yml' + - 'docker-compose.yaml' jobs: