create-public-files-0.0.1 #18
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: create-public-files | |
| run-name: create-public-files-${{ inputs.version }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Input values must match those in params.yaml | |
| dataset: | |
| required: true | |
| description: Comma-separated list of datasets | |
| default: 'times,points,missing_pairs,metadata' | |
| type: string | |
| version: | |
| required: true | |
| description: Version of data | |
| default: '0.0.1' | |
| type: string | |
| mode: | |
| required: true | |
| description: Comma-separated list of travel modes | |
| default: 'auto,bicycle,pedestrian' | |
| type: string | |
| year: | |
| required: true | |
| description: Comma-separated list of years | |
| default: '2020,2021,2022,2023,2024' | |
| type: string | |
| geography: | |
| required: true | |
| description: Comma-separated list of Census geographies | |
| default: 'state,county,county_subdivision,tract,block_group,zcta' | |
| type: string | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| # See: https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 | |
| AWS_EC2_METADATA_DISABLED: true | |
| PYTHONUNBUFFERED: "1" | |
| jobs: | |
| setup-jobs: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| modes: ${{ steps.parse-inputs.outputs.modes }} | |
| years: ${{ steps.parse-inputs.outputs.years }} | |
| geographies: ${{ steps.parse-inputs.outputs.geographies }} | |
| steps: | |
| - name: Parse inputs | |
| id: parse-inputs | |
| shell: bash | |
| run: | | |
| echo "modes=$(echo '${{ inputs.mode }}' | jq -R -c 'split(",")')" >> "$GITHUB_OUTPUT" | |
| echo "years=$(echo '${{ inputs.year }}' | jq -R -c 'split(",")')" >> "$GITHUB_OUTPUT" | |
| echo "geographies=$(echo '${{ inputs.geography }}' | jq -R -c 'split(",")')" >> "$GITHUB_OUTPUT" | |
| create-files: | |
| runs-on: ubuntu-24.04 | |
| needs: setup-jobs | |
| name: create-file-${{ inputs.version }}-${{ matrix.mode }}-${{ matrix.year }}-${{ matrix.geography }} | |
| strategy: | |
| # Don't fail all chunks if one fails | |
| fail-fast: false | |
| matrix: | |
| mode: ${{ fromJSON(needs.setup-jobs.outputs.modes) }} | |
| year: ${{ fromJSON(needs.setup-jobs.outputs.years) }} | |
| geography: ${{ fromJSON(needs.setup-jobs.outputs.geographies) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - name: Install Python dependencies | |
| id: install-python-dependencies | |
| shell: bash | |
| run: pip install .[site] .[data] | |
| - name: Setup Cloudflare credentials | |
| uses: ./.github/actions/setup-cloudflare-s3 | |
| with: | |
| CLOUDFLARE_S3_API_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_S3_API_ACCESS_KEY_ID }} | |
| CLOUDFLARE_S3_API_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_S3_API_SECRET_ACCESS_KEY }} | |
| - name: Create public files | |
| id: create-public-files | |
| working-directory: 'data' | |
| shell: bash | |
| run: | | |
| datasets_parsed=($(echo "${{ inputs.dataset }}" | tr -d ' ' | tr ',' ' ')) | |
| for dataset in "${datasets_parsed[@]}"; do | |
| python ./src/create_public_files.py \ | |
| --dataset "$dataset" --version ${{ inputs.version }} \ | |
| --mode ${{ matrix.mode }} --year ${{ matrix.year }} \ | |
| --geography ${{ matrix.geography }} | |
| done |