Skip to content

feat(plots): plot inferred time-clonal fate potency correlation #1123

feat(plots): plot inferred time-clonal fate potency correlation

feat(plots): plot inferred time-clonal fate potency correlation #1123

Workflow file for this run

name: CID
on:
push:
branches:
- main
- beta
paths-ignore:
- ".github/workflows/colab.yml"
- ".github/workflows/manuscript.yaml"
- "conda/**"
- "dev/**"
- "docs/**"
- "reproducibility/**"
- "scripts/**"
- "**.md"
- "*"
- "!pyproject.toml"
- "!poetry.lock"
- "!flake.nix"
- "!flake.lock"
pull_request:
types: [opened, synchronize, labeled, reopened]
paths-ignore:
- ".github/workflows/colab.yml"
- ".github/workflows/manuscript.yaml"
- "conda/**"
- "dev/**"
- "docs/**"
- "reproducibility/**"
- "scripts/**"
- "**.md"
- "*"
- "!pyproject.toml"
- "!poetry.lock"
- "!flake.nix"
- "!flake.lock"
workflow_dispatch:
inputs:
debug_enabled:
description: "Run with tmate.io debugging enabled"
required: true
type: boolean
default: false
run_build_images:
description: "Run build-images job"
required: false
type: boolean
default: false
run_execute_workflow:
description: "Run execute-workflow job"
required: false
type: boolean
default: false
workflow_execution_mode:
description: "Workflow execution mode"
required: false
type: string
default: "prod"
defaults:
run:
shell: bash
permissions:
contents: read
packages: write
attestations: write
actions: write
id-token: write
jobs:
set-variables:
runs-on: ubuntu-latest
outputs:
debug: ${{ steps.set-variables.outputs.debug }}
skip_ci: ${{ steps.set-variables.outputs.skip_ci }}
skip_tests: ${{ steps.set-variables.outputs.skip_tests }}
mode: ${{ steps.set-variables.outputs.mode }}
checkout_ref: ${{ steps.set-variables.outputs.checkout_ref }}
checkout_rev: ${{ steps.set-variables.outputs.checkout_rev }}
steps:
- name: Set action variables
id: set-variables
run: |
DEBUG="false"
MODE="prod"
SKIP_CI="false"
SKIP_TESTS="false"
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
DEBUG="${{ inputs.debug_enabled }}"
MODE="${{ inputs.workflow_execution_mode }}"
fi
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
if ${{ contains(github.event.pull_request.labels.*.name, 'skip-ci') }}; then
SKIP_CI="true"
fi
if ${{ contains(github.event.pull_request.labels.*.name, 'skip-tests') }}; then
SKIP_TESTS="true"
fi
if ${{ contains(github.event.pull_request.labels.*.name, 'actions-debug') }}; then
DEBUG="true"
fi
if ${{ contains(github.event.pull_request.labels.*.name, 'dev-mode') }}; then
MODE="dev"
fi
CHECKOUT_REF="${{ github.event.pull_request.head.ref }}"
CHECKOUT_REV="${{ github.event.pull_request.head.sha }}"
else
CHECKOUT_REF="${{ github.ref_name }}"
CHECKOUT_REV="${{ github.sha }}"
fi
echo "DEBUG=$DEBUG"
echo "MODE=$MODE"
echo "SKIP_CI=$SKIP_CI"
echo "SKIP_TESTS=$SKIP_TESTS"
echo "CHECKOUT_REF=$CHECKOUT_REF"
echo "CHECKOUT_REV=$CHECKOUT_REV"
echo "DEBUG=$DEBUG" >> $GITHUB_OUTPUT
echo "MODE=$MODE" >> $GITHUB_OUTPUT
echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
echo "SKIP_TESTS=$SKIP_TESTS" >> $GITHUB_OUTPUT
echo "CHECKOUT_REF=$CHECKOUT_REF" >> $GITHUB_OUTPUT
echo "CHECKOUT_REV=$CHECKOUT_REV" >> $GITHUB_OUTPUT
config-workflows:
needs: [set-variables]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' }}
runs-on: ubuntu-latest
outputs:
config-path: ${{ steps.config-output.outputs.path }}
steps:
- name: Checkout code
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- name: Create Flyte config from YAML template
id: yq-process
uses: mikefarah/yq@ef6fb92e7f314e7f0ef49da4385458271203119a # ratchet:mikefarah/yq@master
with:
cmd: "yq e \
'.admin.endpoint = strenv(FLYTE_CLUSTER_ENDPOINT) | \
.storage.stow.config.project_id = strenv(GCP_PROJECT_ID) | \
.storage.stow.config.scopes = strenv(GCP_STORAGE_SCOPES) | \
.storage.container = strenv(GCP_STORAGE_CONTAINER)' \
.flyte/config-template.yaml > .flyte/config.yaml"
env:
FLYTE_CLUSTER_ENDPOINT: ${{ secrets.FLYTE_CLUSTER_ENDPOINT }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
GCP_STORAGE_SCOPES: ${{ secrets.GCP_STORAGE_SCOPES }}
GCP_STORAGE_CONTAINER: ${{ secrets.GCP_STORAGE_CONTAINER }}
- name: Upload Flyte config as an artifact
id: config-output
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
with:
name: flyte-config
path: ${{ secrets.FLYTECTL_CONFIG }}
test-python:
runs-on: ubuntu-latest
needs: [set-variables, config-workflows]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
concurrency:
group: test-python-${{ matrix.python_version }}-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }}-${{ needs.set-variables.outputs.mode }}
cancel-in-progress: true
strategy:
fail-fast: false
matrix:
include:
- python_version: "3.11"
continue-on-error: false
- python_version: "3.12"
continue-on-error: true
steps:
- name: Check Variables
run: |
echo "SKIP_CI=${{ needs.set-variables.outputs.skip_ci }}"
echo "SKIP_TESTS=${{ needs.set-variables.outputs.skip_tests }}"
echo "DEBUG=${{ needs.set-variables.outputs.debug }}"
echo "MODE=${{ needs.set-variables.outputs.mode }}"
- name: Checkout code
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- name: Setup environment
uses: ./.github/actions/setup_environment
with:
python_version: ${{ matrix.python_version }}
debug_enabled: ${{ needs.set-variables.outputs.debug }}
continue-on-error: ${{ matrix.continue-on-error }}
- name: Download Flyte config
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
name: flyte-config
path: .flyte/
continue-on-error: ${{ matrix.continue-on-error }}
- name: "Setup tmate debug session"
uses: mxschmitt/action-tmate@a283f9441d2d96eb62436dc46d7014f5d357ac22 # ratchet:mxschmitt/action-tmate@v3
if: ${{ inputs.debug_enabled }}
- name: Lint and typecheck
run: |
make lint-check
continue-on-error: ${{ matrix.continue-on-error }}
- name: Run tests
run: |
make test-cov-xml
continue-on-error: ${{ matrix.continue-on-error }}
- name: Upload coverage
uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false
verbose: true
continue-on-error: ${{ matrix.continue-on-error }}
test-bazel:
runs-on: ubuntu-latest
needs: [set-variables, config-workflows]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
concurrency:
group: test-bazel-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }}-${{ needs.set-variables.outputs.mode }}
cancel-in-progress: true
permissions:
contents: read
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- name: setup gcp credentials
run: |
cat > service-account-credentials.json << EOF
${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}
EOF
- name: cache bazel
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # ratchet:actions/cache@v4
with:
path: |
~/.cache/bazel
key: ${{ runner.os }}-bazel-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}
restore-keys: |
${{ runner.os }}-bazel-
- name: cache requirements
id: cache-requirements
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # ratchet:actions/cache@v4
with:
path: |
requirements-bazel.txt
key: ${{ runner.os }}-requirements-${{ hashFiles('requirements-cpu.txt', 'MODULE.bazel') }}
restore-keys: |
${{ runner.os }}-requirements-
- name: update linux requirements
if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }}
run: |
make lock-bazel
- name: run bazel tests
run: |
df -h
make meta-bazel test-bazel
df -h
- name: upload requirements artifact
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
with:
name: requirements-bazel
path: requirements-bazel.txt
test-nix:
runs-on: ubuntu-latest
# runs-on: pinellolab-runners
needs: [set-variables, config-workflows]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
concurrency:
group: test-nix-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }}-${{ needs.set-variables.outputs.mode }}
cancel-in-progress: true
permissions:
contents: read
steps:
- name: Maximize build space
# uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c # ratchet:easimon/maximize-build-space@v10
uses: cameronraysmith/maximize-build-space@fdf0c06b18d92be98aa64cb68ae4ea4c9bc4794d # ratchet:cameronraysmith/maximize-build-space@print-usage
with:
build-mount-path: /nix
build-mount-path-ownership: root:root
root-reserve-mb: 34816
swap-size-mb: 1024
remove-dotnet: "true"
remove-android: "true"
remove-codeql: "true"
remove-docker-images: "true"
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- uses: DeterminateSystems/nix-installer-action@7993355175c2765e5733dae74f3e0786fe0e5c4f # ratchet:DeterminateSystems/nix-installer-action@v12
# - uses: DeterminateSystems/magic-nix-cache-action@b46e247b898aa56e6d2d2e728dc6df6c84fdb738 # ratchet:DeterminateSystems/magic-nix-cache-action@v7
# # Toggling use-gha-cache can help resolve `Nix daemon disconnected unexpectedly`.
# with:
# use-gha-cache: false
- uses: cachix/cachix-action@ad2ddac53f961de1989924296a1f236fcfbaa4fc # ratchet:cachix/cachix-action@v15
with:
name: "${{ vars.CACHIX_CACHE_NAME }}"
authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
# - name: "Authenticate to Google Cloud"
# uses: "google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c" # ratchet:google-github-actions/auth@v2
# with:
# credentials_json: "${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}"
# - name: Set up remote cache
# uses: zombiezen/setup-nix-cache-action@78efa9db5802d6a141bc42681edf000bf7f31a08 # ratchet:zombiezen/setup-nix-cache-action@v0
# with:
# substituters: "${{ vars.NIX_SUBSTITUTER }}"
# secret_keys: "${{ secrets.NIX_PRIVATE_KEY }}"
# use_nixcached: true
# nixcached_upload_options: --jobs 8
- uses: rlespinasse/github-slug-action@797d68864753cbceedc271349d402da4590e6302 # ratchet:rlespinasse/github-slug-action@v4
with:
prefix: CI_
- name: Set git variables
run: |
echo "GIT_REPO_NAME=$CI_GITHUB_REPOSITORY_NAME_PART" >> $GITHUB_ENV
echo "GIT_REF=$CI_GITHUB_REF_NAME" >> $GITHUB_ENV
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "GIT_SHA=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> $GITHUB_ENV
echo "GIT_SHA_SHORT=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT" >> $GITHUB_ENV
else
echo "GIT_SHA=$CI_GITHUB_SHA" >> $GITHUB_ENV
echo "GIT_SHA_SHORT=$CI_GITHUB_SHA_SHORT" >> $GITHUB_ENV
fi
- name: Check nix flake
run: nix flake check --impure --accept-flake-config
# ./scripts/flake can be used to enable the flake for testing
# - run: |
# ./scripts/flake pytest
- name: Test nix package
run: |
set -euo pipefail
df -h
nix build .#default \
--accept-flake-config \
--impure \
--fallback \
--keep-going \
--print-build-logs
df -h
build-nix-images:
needs: [set-variables]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && ( contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images) || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow) ) }}
uses: ./.github/workflows/build-nix-images.yaml
concurrency:
group: bni-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }}-${{ needs.set-variables.outputs.mode }}-${{ github.sha }}
cancel-in-progress: true
with:
debug_enabled: ${{ needs.set-variables.outputs.debug }}
version: ""
images: '["pyrovelocity"]'
branch: ${{ needs.set-variables.outputs.checkout_ref }}
revision: ${{ needs.set-variables.outputs.checkout_rev }}
mode: ${{ needs.set-variables.outputs.mode }}
execute-workflow:
needs: [config-workflows, build-nix-images, set-variables]
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow)) }}
runs-on: ubuntu-latest
concurrency:
group: ef-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }}-${{ needs.set-variables.outputs.mode }}-${{ github.sha }}
cancel-in-progress: true
strategy:
matrix:
python_version: ["3.11"]
env:
FLYTECTL_CONFIG: ${{ secrets.FLYTECTL_CONFIG }}
FLYTE_OAUTH_CLIENT_SECRET: ${{ secrets.FLYTE_OAUTH_CLIENT_SECRET }}
WORKFLOW_IMAGE: ${{ vars.WORKFLOW_IMAGE }}
steps:
- name: Checkout code
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- name: Setup environment
uses: ./.github/actions/setup_environment
with:
python_version: ${{ matrix.python_version }}
debug_enabled: ${{ needs.set-variables.outputs.debug }}
- name: Download Flyte config
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
name: flyte-config
path: .flyte/
- name: Setup tmate debug session
if: ${{ inputs.debug_enabled == 'true' }}
uses: mxschmitt/action-tmate@a283f9441d2d96eb62436dc46d7014f5d357ac22 # ratchet:mxschmitt/action-tmate@v3
- name: Execute workflow
id: execute
run: |
make run-${{ needs.set-variables.outputs.mode }}
- name: Create config tarball
id: save-hydra-outputs
run: |
TAR_FILENAME="hydra_outputs_${{ needs.set-variables.outputs.checkout_rev }}.tar.gz"
tar -czf $TAR_FILENAME ./outputs/
tar -tzf $TAR_FILENAME
echo "HYDRA_OUTPUTS_TAR=$TAR_FILENAME" >> $GITHUB_ENV
- name: Upload config artifact
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
with:
name: hydra-outputs
path: ${{ env.HYDRA_OUTPUTS_TAR }}
release:
runs-on: ubuntu-latest
needs: [test-python, test-nix, test-bazel]
if: ${{ github.repository_owner == 'pinellolab' && github.event_name == 'push' }}
environment:
name: release
url: https://github.com/pinellolab/pyrovelocity/releases/tag/${{ steps.semanticrelease.outputs.git-tag }}
permissions:
contents: write
outputs:
version: ${{ steps.semanticrelease.outputs.version }}
released: ${{ steps.semanticrelease.outputs.released }}
git-head: ${{ steps.semanticrelease.outputs.git-head }}
git-tag: ${{ steps.semanticrelease.outputs.git-tag }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- uses: DeterminateSystems/nix-installer-action@7993355175c2765e5733dae74f3e0786fe0e5c4f # ratchet:DeterminateSystems/nix-installer-action@v12
# - uses: DeterminateSystems/magic-nix-cache-action@b46e247b898aa56e6d2d2e728dc6df6c84fdb738 # ratchet:DeterminateSystems/magic-nix-cache-action@v7
- uses: cachix/cachix-action@ad2ddac53f961de1989924296a1f236fcfbaa4fc # ratchet:cachix/cachix-action@v15
with:
name: "${{ vars.CACHIX_CACHE_NAME }}"
authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
# - uses: "google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c" # ratchet:google-github-actions/auth@v2
# with:
# credentials_json: "${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}"
# - uses: zombiezen/setup-nix-cache-action@78efa9db5802d6a141bc42681edf000bf7f31a08 # ratchet:zombiezen/setup-nix-cache-action@v0
# with:
# substituters: "${{ vars.NIX_SUBSTITUTER }}"
# secret_keys: "${{ secrets.NIX_PRIVATE_KEY }}"
# use_nixcached: true
# nixcached_upload_options: --jobs 8
- run: nix profile install .#releaseEnv
- uses: cihelper/action-semanticrelease-poetry@66900ba780321bcdb974891ee82f945a25a870d7 # ratchet:cihelper/action-semanticrelease-poetry@v1
id: semanticrelease
with:
github-token: ${{ secrets.FAST_FORWARD_PAT }}
- uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
if: ${{ steps.semanticrelease.outputs.released == 'true' }}
with:
name: poetry-build
path: ./dist
publish-nix-flake:
runs-on: ubuntu-latest
needs: [set-variables, release]
if: ${{ needs.release.outputs.released == 'true' }}
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.release.outputs.git-tag }}
- uses: DeterminateSystems/nix-installer-action@7993355175c2765e5733dae74f3e0786fe0e5c4f # ratchet:DeterminateSystems/nix-installer-action@v12
- uses: DeterminateSystems/flakehub-push@d3b4b64eb6ff61361ac7a7ac146eb443c86921a2 # ratchet:DeterminateSystems/flakehub-push@main
with:
visibility: unlisted
tag: ${{ needs.release.outputs.version }}
publish-release-images:
needs: [set-variables, release]
if: ${{ needs.release.outputs.released == 'true' }}
uses: ./.github/workflows/build-nix-images.yaml
concurrency:
group: pri-${{ github.workflow }}-${{ github.ref_name }}-${{ github.sha }}
cancel-in-progress: true
with:
debug_enabled: ${{ needs.set-variables.outputs.debug }}
version: ${{ needs.release.outputs.version }}
images: '["pyrovelocity","pyrovelocitydev","pyrovelocitycode","pyrovelocityjupyter"]'
branch: ${{ needs.set-variables.outputs.checkout_ref }}
revision: ${{ needs.release.outputs.git-head }}
mode: "prod"
publish-pypi:
runs-on: ubuntu-latest
needs: release
if: ${{ needs.release.outputs.released == 'true' }}
permissions:
id-token: write
environment:
name: release
url: https://pypi.org/project/pyrovelocity/${{needs.release.outputs.version}}/
steps:
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
name: poetry-build
path: ./dist
- uses: pypa/gh-action-pypi-publish@fb9fc6a4e67ca27a7a76b17bbf90be83c2d3c716 # ratchet:pypa/gh-action-pypi-publish@release/v1
test-docs-build:
runs-on: ubuntu-latest
needs: set-variables
if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
strategy:
matrix:
python_version: ["3.11"]
permissions:
contents: read
pages: read
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.set-variables.outputs.checkout_ref }}
- name: Setup environment
uses: ./.github/actions/setup_environment
with:
python_version: ${{ matrix.python_version }}
debug_enabled: ${{ needs.set-variables.outputs.debug }}
- name: Build
run: make docs-build
build-release-docs:
runs-on: ubuntu-latest
needs: [set-variables, release]
if: ${{ needs.release.outputs.released == 'true' }}
strategy:
matrix:
python_version: ["3.11"]
permissions:
contents: read
pages: read
environment: github-pages
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
ref: ${{ needs.release.outputs.git-tag }}
- name: Setup environment
uses: ./.github/actions/setup_environment
with:
python_version: ${{ matrix.python_version }}
debug_enabled: ${{ needs.set-variables.outputs.debug }}
- name: Build
run: make docs-build
- name: Upload artifact
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # ratchet:actions/upload-pages-artifact@v3
with:
path: ./site
publish-docs:
runs-on: ubuntu-latest
needs: [build-release-docs, release]
if: ${{ needs.release.outputs.released == 'true' }}
permissions:
id-token: write
pages: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # ratchet:actions/deploy-pages@v4