-
-
Notifications
You must be signed in to change notification settings - Fork 1
199 lines (197 loc) · 9.94 KB
/
workflow_call.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
name: Open notebooks in Colab
on:
workflow_call:
inputs:
work_directory:
description: "Work directory with a copy of the files to be processed"
type: string
notebook_pattern:
description: "Search pattern for notebooks to be processed"
type: string
extra_files_pattern:
description: "Search pattern for extra files to be processed"
type: string
notebook_preparation:
description: "Script to be run before processing notebooks"
type: string
extra_files_preparation:
description: "Script to be run before processing extra files"
type: string
test_script:
description: "Script to be run to test notebooks"
type: string
fem_on_colab_packages:
description: "List of packages that need an installation cell among the one offered by FEM on Colab"
type: string
pip_packages:
description: "List of pip-installable packages that need an installation cell"
type: string
publish_on:
description: "How to publish the processed notebooks"
type: string
publish_if_repository:
description: "Restrict publishing to a specific calling repository"
type: string
publish_if_branch:
description: "Restrict publishing to a specific branch on the calling repository"
type: string
secrets:
RCLONE_CONFIG_DRIVE_CLIENT_ID:
description: "Client ID for the Google Acccount. Only used when publish_on is drive"
RCLONE_CONFIG_DRIVE_CLIENT_SECRET:
description: "Secret for the Google Acccount. Only used when publish_on is drive"
RCLONE_CONFIG_DRIVE_TOKEN:
description: "Token for the Google Acccount. Only used when publish_on is drive"
REPO_ACCESS_TOKEN:
description: "Token that enables writing to the destination repository. Only used when publish_on is github"
jobs:
run:
runs-on: ubuntu-latest
container: femoncolab/base_images:latest
steps:
- name: Mark workspace as safe
run: |
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
- uses: actions/checkout@v4
with:
set-safe-directory: false
- name: Fetch updated package list
run: apt update -y -q
- name: Install non-pip dependencies of the workflow call library
run: |
apt install -y -qq imagemagick inkscape
wget https://github.com/rclone/rclone/releases/download/v1.68.1/rclone-v1.68.1-linux-amd64.deb -O /tmp/rclone-v1.68.1-linux-amd64.deb
dpkg -i /tmp/rclone-v1.68.1-linux-amd64.deb
- name: Determine which branch to use while cloning the workflow call library
id: workflow_call_library_branch
run: |
if [[ "${{ github.repository }}" == "fem-on-colab/open-in-colab-workflow" ]]; then
BRANCH=${GITHUB_REF##*/}
else
BRANCH="main"
fi
echo "branch=${BRANCH}" >> ${GITHUB_OUTPUT}
- name: Clone workflow call library
uses: actions/checkout@v4
with:
repository: fem-on-colab/open-in-colab-workflow.git
ref: ${{ steps.workflow_call_library_branch.outputs.branch }}
path: _workflow_call_library
- name: Install the workflow call library
run: |
pushd _workflow_call_library
python3 -m pip install .[docs,lint,tests]
popd
rm -rf _workflow_call_library
shell: bash
- name: Determine publisher settings
id: determine_publisher
run: |
python3 -m open_in_cloud_workflow.publish_on "${{ inputs.publish_on }}" > .publisher
while read -r VARIABLE_VALUE; do
VARIABLE_VALUE_ARRAY=(${VARIABLE_VALUE//=/ })
echo "${VARIABLE_VALUE_ARRAY[0]}=${VARIABLE_VALUE_ARRAY[1]}" >> ${GITHUB_OUTPUT}
done < .publisher
rm .publisher
shell: bash
- name: Mark work directory as safe
if: steps.determine_publisher.outputs.publisher == 'github' && (github.repository == inputs.publish_if_repository || inputs.publish_if_repository == '') && (endsWith(github.ref, inputs.publish_if_branch) || inputs.publish_if_branch == '')
run: |
git config --global --add safe.directory "${{ inputs.work_directory }}"
- name: Clone GitHub repository of the publisher
if: steps.determine_publisher.outputs.publisher == 'github' && (github.repository == inputs.publish_if_repository || inputs.publish_if_repository == '') && (endsWith(github.ref, inputs.publish_if_branch) || inputs.publish_if_branch == '')
uses: actions/checkout@v4
with:
repository: "${{ steps.determine_publisher.outputs.repository }}"
token: "${{ secrets.REPO_ACCESS_TOKEN || github.token }}"
ref: "${{ steps.determine_publisher.outputs.branch }}"
fetch-depth: 0
path: "${{ inputs.work_directory }}"
set-safe-directory: false
- name: Run notebook preparation
if: inputs.notebook_preparation != ''
run: |
${{ inputs.notebook_preparation }}
shell: bash
- name: Run extra files preparation
if: inputs.extra_files_preparation != ''
run: |
${{ inputs.extra_files_preparation }}
- name: Add installation cells
run: |
python3 -m open_in_cloud_workflow.add_installation_cells "${PWD}/${{ inputs.work_directory }}" "${{ inputs.notebook_pattern }}" "colab" '${{ inputs.fem_on_colab_packages }}' '${{ inputs.pip_packages }}'
- name: Test notebooks in the work directory
if: inputs.test_script != ''
run: |
export PYTHONPATH="/usr/lib/${PYTHON_VERSION}/test-task/extra-site-packages:${PYTHONPATH}"
export LD_LIBRARY_PATH=""
${{ inputs.test_script }}
shell: bash
- name: Replace images in markdown
run: |
python3 -m open_in_cloud_workflow.replace_images_in_markdown "${PWD}/${{ inputs.work_directory }}" "${{ inputs.notebook_pattern }}"
- name: Replace links in markdown
run: |
python3 -m open_in_cloud_workflow.replace_links_in_markdown "${PWD}/${{ inputs.work_directory }}" "${{ inputs.notebook_pattern }}" "colab" "${{ inputs.publish_on }}"
env:
RCLONE_CONFIG_DRIVE_CLIENT_ID: "${{ secrets.RCLONE_CONFIG_DRIVE_CLIENT_ID }}"
RCLONE_CONFIG_DRIVE_CLIENT_SECRET: "${{ secrets.RCLONE_CONFIG_DRIVE_CLIENT_SECRET }}"
RCLONE_CONFIG_DRIVE_TOKEN: "${{ secrets.RCLONE_CONFIG_DRIVE_TOKEN }}"
- name: Determine publisher upload pattern
if: success() || (steps.determine_publisher.outputs.publisher == 'artifact' && (failure() || cancelled()))
id: upload_pattern
run: |
if [[ "${{ steps.determine_publisher.outputs.publisher }}" == "artifact" ]]; then
UPLOAD_PATTERN_PREFIX="${{ inputs.work_directory }}/"
else
UPLOAD_PATTERN_PREFIX=""
fi
UPLOAD_PATTERN_ARRAY=()
while read -r PATTERN; do
UPLOAD_PATTERN_ARRAY+=("${UPLOAD_PATTERN_PREFIX}${PATTERN}")
done <<< "${{ inputs.notebook_pattern }}"
if [[ -n "${{ inputs.extra_files_pattern }}" ]]; then
while read -r PATTERN; do
UPLOAD_PATTERN_ARRAY+=("${UPLOAD_PATTERN_PREFIX}${PATTERN}")
done <<< "${{ inputs.extra_files_pattern }}"
fi
SEPARATOR="%0A"
UPLOAD_PATTERN="$(printf "${SEPARATOR//%/%%}%s" "${UPLOAD_PATTERN_ARRAY[@]}")"
UPLOAD_PATTERN="${UPLOAD_PATTERN:${#SEPARATOR}}"
echo "upload_pattern_separator=${SEPARATOR}" >> ${GITHUB_OUTPUT}
echo "upload_pattern=${UPLOAD_PATTERN}" >> ${GITHUB_OUTPUT}
shell: bash
- name: Upload files to an artifact
if: steps.determine_publisher.outputs.publisher == 'artifact' && (success() || failure() || cancelled())
uses: actions/upload-artifact@v4
with:
name: ${{ steps.determine_publisher.outputs.name }}
path: ${{ steps.upload_pattern.outputs.upload_pattern }}
- name: Upload files to Google Drive
if: steps.determine_publisher.outputs.publisher == 'drive' && (github.repository == inputs.publish_if_repository || inputs.publish_if_repository == '') && (endsWith(github.ref, inputs.publish_if_branch) || inputs.publish_if_branch == '')
run: |
PATTERNS=$(echo "${{ steps.upload_pattern.outputs.upload_pattern }}" | sed "s/${{ steps.upload_pattern.outputs.upload_pattern_separator }}/\n/g")
python3 -m open_in_cloud_workflow.upload_files_to_google_drive "${{ inputs.work_directory }}" "${PATTERNS}" "${{ inputs.publish_on }}"
env:
RCLONE_CONFIG_DRIVE_CLIENT_ID: "${{ secrets.RCLONE_CONFIG_DRIVE_CLIENT_ID }}"
RCLONE_CONFIG_DRIVE_CLIENT_SECRET: "${{ secrets.RCLONE_CONFIG_DRIVE_CLIENT_SECRET }}"
RCLONE_CONFIG_DRIVE_TOKEN: "${{ secrets.RCLONE_CONFIG_DRIVE_TOKEN }}"
- name: Upload files to GitHub repository
if: steps.determine_publisher.outputs.publisher == 'github' && (github.repository == inputs.publish_if_repository || inputs.publish_if_repository == '') && (endsWith(github.ref, inputs.publish_if_branch) || inputs.publish_if_branch == '')
run: |
SHA_SHORT=$(git rev-parse --short HEAD)
pushd "${{ inputs.work_directory }}"
PATTERNS=($(echo "${{ steps.upload_pattern.outputs.upload_pattern }}" | sed "s/${{ steps.upload_pattern.outputs.upload_pattern_separator }}/ /g"))
for PATTERN in "${PATTERNS[@]}"; do
git add "${PATTERN}"
done
if [[ "$(git diff --name-only --cached | wc -l)" -gt 0 ]]; then
git config user.name "GitHub Actions"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
BRANCH="${{ steps.determine_publisher.outputs.branch }}"
git pull origin ${BRANCH}
git commit -m "deploy: ${GITHUB_REPOSITORY}@${SHA_SHORT}"
git push origin ${BRANCH}
fi
popd
shell: bash