create-valhalla-tiles.yaml

---

name: create-valhalla-tiles
run-name: create-valhalla-tiles-${{ inputs.year }}

on:
  workflow_dispatch:
    inputs:
      year:
        required: true
        description: OSM data year
        default: '2020'
        type: choice
        options:
          - '2020'
          - '2021'
          - '2022'
          - '2023'
          - '2024'

      override_states:
        required: false
        description: |
          Comma-separated state codes to run e.g. 01,06.
          Will run all if null
        type: string

env:
  AWS_DEFAULT_REGION: us-east-1
  # See: https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151
  AWS_EC2_METADATA_DISABLED: true
  PYTHONUNBUFFERED: "1"

jobs:
  setup-jobs:
    runs-on: ubuntu-24.04

    outputs:
      states: ${{ steps.create-job-chunks.outputs.states }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Build Dockerized dependencies
        uses: ./.github/actions/build-docker
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Create job chunks
        id: create-job-chunks
        shell: bash
        run: |
          states=$(yq e -o=json '.input.state' ./data/params.yaml | jq -c -s .[])
          echo "states=$(echo $states)" >> $GITHUB_OUTPUT

          # If override states are set, use those instead
          states_parsed=($(echo "$states" | jq -r '.[]'))
          if [ -n "${{ inputs.override_states }}" ]; then
            override_states_parsed=($(echo "${{ inputs.override_states }}" | tr -d ' ' | tr ',' ' '))
            for state in "${override_states_parsed[@]}"; do
              if [[ ! " ${states_parsed[@]} " =~ " ${state} " ]]; then
                echo "Error: Override state ${state} is not in the states for this workflow"
                echo "States include: ${states_parsed[@]}"
                exit 1
              fi
            done
            states_json=$(printf '%s\n' "${override_states_parsed[@]}" | jq -c -R . | jq -c -s .)
            echo "Creating jobs for states: ${override_states_parsed[@]}"
            echo "states=$states_json" > $GITHUB_OUTPUT
          else
            echo "Creating jobs for states: ${states_parsed[@]}"
          fi

  run-job:
    runs-on: ubuntu-24.04
    needs: setup-jobs
    strategy:
      # Don't fail all chunks if one fails
      fail-fast: false
      matrix:
        state: ${{ fromJSON(needs.setup-jobs.outputs.states) }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Cloudflare credentials
        uses: ./.github/actions/setup-cloudflare-s3
        with:
          CLOUDFLARE_S3_API_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_S3_API_ACCESS_KEY_ID }}
          CLOUDFLARE_S3_API_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_S3_API_SECRET_ACCESS_KEY }}

      - name: Install DVC
        uses: ./.github/actions/setup-dvc

      - name: Fetch Docker image
        uses: actions/download-artifact@v4
        with:
          name: opentimes-docker-${{ hashFiles('./data/Dockerfile', './pyproject.toml') }}
          path: /tmp

      - name: Load Docker image
        run: |
          docker load --input /tmp/opentimes.tar
          docker image ls -a

      - name: Pull DVC objects
        shell: bash
        working-directory: 'data'
        run: |
          dvc pull --no-run-cache \
            ./intermediate/osmextract/year=${{ inputs.year }}/geography=state/state=${{ matrix.state }}/${{ matrix.state }}.osm.pbf

      - name: Run job chunk
        shell: bash
        working-directory: 'data'
        env:
          # Disable elevation for Alaska (which requires 22GB of tiles)
          BUILD_ELEVATION: ${{ matrix.state == '02' && 'False' || 'True' }}
        run: |
          dvc repro -s create_valhalla_tiles@${{ inputs.year }}-${{ matrix.state }}

      - name: Write tile files to S3
        shell: bash
        working-directory: 'data'
        run: |
          aws s3 cp --quiet --endpoint-url \
            https://${{ vars.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com \
             ./intermediate/valhalla_tiles/year=${{ inputs.year }}/geography=state/state=${{ matrix.state }}/valhalla_tiles.tar.zst \
             s3://opentimes-resources/valhalla_tiles/year=${{ inputs.year }}/geography=state/state=${{ matrix.state }}/valhalla_tiles.tar.zst \
            --profile cloudflare