.gitlab-ci.yml

# 
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2024, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

default:
  tags:
    - ubuntu_22_04_amd64
  # cancel a job if a new pipeline is triggered on the same branch
  interruptible: true

# Required for Manual runs through UI
variables:
  PYVERSION:
    value: 11
    description: Default version of Python to use for non-test jobs

  WEB_SKIP_TEST:
    value: "False"
    options:
      - "True"
      - "False"
    description: True to skip test job when triggered via Web UI.
  WEB_SKIP_AUTOBRANCH:
    value: "False"
    options:
      - "True"
      - "False"
    description: True to skip autobranch job when triggered via Web UI.
  WEB_SKIP_PAGES:
    value: "False"
    options:
      - "True"
      - "False"
    description: True to skip pages job when triggered via Web UI.

Test:
  rules:
      - if: $CI_PIPELINE_SOURCE == "web"
      - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  parallel:
    matrix:
      - PYVERSION: [9, 11]
  before_script:
    # Make the test a no-op so we can run any step depending on successful tests
    - if [[ "$CI_PIPELINE_SOURCE" == "web" && "$WEB_SKIP_TEST" == "True" ]]; then exit 0; fi

    - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads'
    - export DEBIAN_FRONTEND=noninteractive
    - sudo apt-get update -y
    - sudo apt-get upgrade -y
    - sudo apt-get install software-properties-common -y
    - sudo add-apt-repository ppa:deadsnakes/ppa -y
    - sudo apt-get update -y
    - sudo apt-get -y install python3.${PYVERSION}-full
    - sudo chmod o+rx /sys/kernel/debug
  script:
    - export LISA_PYTHON=python3.${PYVERSION}
    - sudo ./install_base.sh --install-all
    - bash ./tools/tests.sh

Autobranch:
  needs: ['Test']
  environment:
    # This grants the GITLAB_REPO_TOKEN variable allowing to push to the repo
    name: repo_write_access

  variables:
    # Do not clone or fetch anything, as we will do it manually so that the job
    # runs with the code from the default branch (main) rather than the merge
    # request branch for safety reasons.
    GIT_STRATEGY: none
  rules:
      - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /(^|,)autobranch($|,)/ 
  resource_group: autobranch
  before_script:
    # Make the test a no-op so we can run any step depending on successful tests
    - if [[ "$CI_PIPELINE_SOURCE" == "web" && "$WEB_SKIP_AUTOBRANCH" == "True" ]]; then exit 0; fi

    - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads'
    - export DEBIAN_FRONTEND=noninteractive
    - sudo apt-get update -y
    - sudo apt-get upgrade -y
    - sudo apt-get install software-properties-common -y
    - sudo add-apt-repository ppa:deadsnakes/ppa -y
    - sudo apt-get update -y
    - sudo apt-get -y install python3.${PYVERSION}-full
    - export LISA_PYTHON=python3.${PYVERSION}

    - |
      # Avoid using /tmp since it's probably in tmpfs and would consume RAM
      export TMPDIR=$(readlink -f .)/tmp
      mkdir -p "$TMPDIR"

      origin_repo=$(mktemp -d)
      git clone -b "$CI_DEFAULT_BRANCH" "$CI_REPOSITORY_URL" "$origin_repo"

      # This is critical for safety: We want to be executing the code from the
      # main branch to avoid any manipulation from the MR branch
      cd "$origin_repo" || exit 1

    - |
      git remote set-url origin "https://gitlab-ci:${GITLAB_REPO_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
      git remote -v
      git branch --list --remotes

      git config --global user.name 'project_444_bot_b970213545be4e6c0185c64ae9fc5226'
      git config --global user.email 'project_444_bot_b970213545be4e6c0185c64ae9fc5226@noreply.gitlab.arm.com'

  script: 
    - |
      function lisa_python() {
        "$LISA_PYTHON" "$@"
      }
      set -e
      export LC_ALL=C
      venv=$(mktemp -d)
      packages=$(mktemp)
      lisa_python -m venv "$venv" && source "$venv/bin/activate"

      # Ensure we have up to date versions of every pre-installed packages
      lisa_python -m pip freeze --exclude-editable --all | cut -f1 -d '=' > "$packages"
      lisa_python -m pip install --upgrade -r "$packages"

      lisa_python -m pip install --upgrade pip-audit ./tools/lisa-combine-gitlab-mr
      pip-audit || exit 1

      function update_branch() {
          local label=$1
          local branch=$2
          local force_branch=$3

          local worktree=$(mktemp -d --suffix "-${branch}-repo")
          local patch=$(mktemp --suffix "-$branch.patch")

          lisa-combine-gitlab-mr --server "$CI_SERVER_HOST" --repo "$CI_PROJECT_PATH" --project-id "$CI_PROJECT_ID" --api-url "$CI_API_V4_URL" --api-token "$GITLAB_REPO_TOKEN" --mr-label "$label" --branch "$force_branch" &&

          git fetch origin "$branch" &&

          # Work in a separate worktree so that there is no risk of folders
          # added to PATH by init_env being manipulated
          git worktree add "$worktree" --checkout "$branch" &&

          git -C "$worktree" diff --binary "HEAD..$force_branch" > "$patch" &&

          if [[ -s "$patch" ]]; then
              # Apply the patch to the index as well, so that any file created
              # is automatically added to the commit we are about to create.
              git -C "$worktree" apply --index "$patch" &&
              git -C "$worktree" commit --all -m "Autocommit to $branch branch on $(date) tracking $force_branch"

              git push --force origin "$force_branch"
              git push origin "$branch"
          else
              echo "Empty patch, $branch and $force_branch branches are up to date."
          fi
      }

      ret=0
      function keepgoing {
          "$@" || ret=1
      }

      keepgoing update_branch for-master-autobranch master master-force
      keepgoing update_branch for-preview-autobranch preview preview-force

      exit $ret


pages:
  resource_group: pages

  artifacts:
    paths:
        - public

  rules:
      - if: $CI_PIPELINE_SOURCE == "web" && $WEB_SKIP_PAGES == "False"
        needs: ['Autobranch']
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"
        needs: ['Autobranch']
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "preview"
        needs: []
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "master"
        needs: []
      - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "release"
        needs: []

  before_script:
    - sudo sh -c 'echo "APT::Acquire::Retries=3;" >> /etc/apt/apt.conf.d/99retry-downloads'
    - export DEBIAN_FRONTEND=noninteractive
    - sudo apt-get update -y
    - sudo apt-get upgrade -y
    - sudo apt-get install software-properties-common symlinks rdfind -y
    - sudo add-apt-repository ppa:deadsnakes/ppa -y
    - sudo apt-get update -y
    - sudo apt-get -y install python3-pip python3.${PYVERSION}-full
    - export LISA_PYTHON=python3.${PYVERSION}

    # This avoids the error: Missing gitlab-runner. Uploading artifacts is disabled.
    # https://docs.gitlab.com/runner/install/linux-repository.html
    - curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash
    - sudo apt-get install gitlab-runner

  script:
    - |
      function lisa_python() {
          "$LISA_PYTHON" "$@"
      }

      function audit() {
          lisa_python -m pip install --upgrade pip-audit
          pip-audit || exit 1
      }

      function upgrade_packages() {
          packages=$(mktemp)
          # Ensure we have up to date versions of every pre-installed packages
          lisa_python -m pip freeze --local --exclude-editable --all | cut -f1 -d '=' > "$packages"
          lisa_python -m pip install --upgrade -r "$packages"
      }

      set -ex
      # Avoid using /tmp since it's probably in tmpfs and would consume RAM
      export TMPDIR=$(readlink -f .)/tmp
      mkdir -p "$TMPDIR"

      df -h .

    - |
      out="$(readlink -f .)/public"

      function build_branch() {
          local ref=$1
          local publish_as=$2
          local worktree=$(mktemp -d --suffix "-worktree")
          git fetch origin "$ref"
          git worktree add "$worktree" --checkout "$ref"
          (
            cd "$worktree"
            sudo ./install_base.sh --install-doc-extras

            set +e
            source ./init_env || return 1
            set -e
            audit

            export LISA_DOC_VERSION="$publish_as"
            export LC_ALL=C
            local doctree=$(mktemp -d --suffix "-doctree")
            cd ./doc && sphinx-build -b html -n --no-color -W --keep-going -T -j auto -d "$doctree" . "$out/$publish_as"
          )
      }

      function fail() {
          if ! "$@"; then
              # If one of the build phases failed, make sure we do not
              # accidentally deploy a partial artifact (likely mostly empty)
              # and exit with error
              rm -r "$out" &>/dev/null
              exit 1
          fi
      }

      export LISA_DOC_BASE_URL="https://tooling.sites.arm.com/lisa"
      # Provide the list of versions for the version switcher
      export LISA_DOC_ALL_VERSIONS='["main", "preview", "master"]'
      fail build_branch main main
      fail build_branch preview preview
      fail build_branch master master

      # TODO: For now the release branch doc won't build as it lacks some necessary fixes
      # build_branch release release
      # build_branch v4.0.0 v4

      # Expose "main" as the "latest" branch
      (cd "$out" && ln -s ./main ./latest)

      # Make the doc a lot more compact by trimming useless files
      find "$out" '-(' -name '*.ipynb' -or -name '*.ipynb.txt' '-)' -delete

      # Replace duplicates with symlinks to compact the archive
      rdfind -makeresultsfile false -makesymlinks true "$out"
      # Rewrite absolute symlinks into relative ones
      symlinks -cr "$out"

      # Redirect bare address such as https://tooling.sites.arm.com/lisa to a
      # given version
      default=latest
      cat > "$out/_redirects" <<- EOF
      /lisa/ /lisa/$default/ 301
      EOF

    # Debug excessive sizes
    - du -hc "$out" | sort -h