From 948da0c8bdc36e5cf183ca6524ff4c30d443e301 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Wed, 28 Oct 2020 00:06:21 +0100 Subject: [PATCH] Add README, FUNDING, and CI files --- .cirrus.yml | 69 +++++ .github/FUNDING.yml | 1 + .github/workflows/main.yml | 56 ++++ .semaphore/semaphore.yml | 48 ++++ README.md | 224 +++++++++++++++ azure-pipelines.yml | 48 ++++ buildci.sh | 542 +++++++++++++++++++++++++++++++++++++ 7 files changed, 988 insertions(+) create mode 100644 .cirrus.yml create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/main.yml create mode 100644 .semaphore/semaphore.yml create mode 100644 README.md create mode 100644 azure-pipelines.yml create mode 100755 buildci.sh diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000000..b63934140981 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,69 @@ +common_tasks_template: &COMMON_TASKS_TEMPLATE + # Location of downloaded prerequesites + gcc_deps_cache: + folder: gcc-deps + fingerprint_script: cat gcc/BASE-VER + # Typical build time is ~25 minutes, factor in twice that for waiting time + timeout_in: 120m + # Scripts to configure, build, and test + setup_script: ./buildci.sh setup + build_script: ./buildci.sh build + test_script: | + if [ "${RUN_TESTSUITE:-0}" = "1" ] + then + ./buildci.sh testsuite + else + ./buildci.sh unittests + fi + # Location of compressed testsuite logs + testsuite_artifacts: + path: logs/** + +environment: + CIRRUS_CLONE_DEPTH: 50 + +# Linux +task: + name: Ubuntu $TASK_NAME_SUFFIX + container: + image: ubuntu:20.04 + cpu: 8 + memory: 16G + environment: + matrix: + - TASK_NAME_SUFFIX: Testsuite + RUN_TESTSUITE: 1 + - TASK_NAME_SUFFIX: Unit Tests + RUN_TESTSUITE: 0 + << : *COMMON_TASKS_TEMPLATE + +# FreeBSD +task: + name: FreeBSD $TASK_NAME_SUFFIX + freebsd_instance: + image_family: freebsd-12-2 + cpu: 8 + memory: 16G + environment: + matrix: + - TASK_NAME_SUFFIX: Testsuite + RUN_TESTSUITE: 1 + - TASK_NAME_SUFFIX: Unit Tests + RUN_TESTSUITE: 0 + install_bash_script: | + pkg install -y bash + ln -s /usr/local/bin/bash /bin/bash + << : *COMMON_TASKS_TEMPLATE + +# Mac +task: + name: Darwin $TASK_NAME_SUFFIX + osx_instance: + image: catalina-xcode + environment: + matrix: + - TASK_NAME_SUFFIX: Testsuite + RUN_TESTSUITE: 1 + - TASK_NAME_SUFFIX: Unit Tests + RUN_TESTSUITE: 0 + << : *COMMON_TASKS_TEMPLATE diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000000..bba80bf6879c --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [ ibuclaw ] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000000..58501a78ba4a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,56 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Main +on: + - push # branch or tag + +jobs: + linux: + strategy: + fail-fast: false + matrix: + include: + - job_name: ubuntu-x86_64 (testsuite) + target: x86_64-linux-gnu + - job_name: ubuntu-x86_64 (unittests) + target: x86_64-linux-gnu + - job_name: ubuntu-x86_64 (bootstrap) + target: x86_64-linux-gnu + bootstrap: enable + - job_name: ubuntu-arm + target: arm-linux-gnueabi + - job_name: ubuntu-armhf + target: arm-linux-gnueabihf + - job_name: ubuntu-aarch64 + target: aarch64-linux-gnu + - job_name: ubuntu-mips + target: mips-linux-gnu + - job_name: ubuntu-mips64el + target: mips64el-linux-gnuabi64 + - job_name: ubuntu-mipsel + target: mipsel-linux-gnu + - job_name: ubuntu-powerpc64le + target: powerpc64le-linux-gnu + - job_name: ubuntu-systemz + target: s390x-linux-gnu + - job_name: ubuntu-sparc64 + target: sparc64-linux-gnu + name: ${{ matrix.job_name }} + runs-on: ubuntu-22.04 + env: + GCC_CI_TARGET: ${{ matrix.target }} + GCC_CI_BOOTSTRAP: ${{ matrix.bootstrap || 'disable' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 + - name: Install dependencies + run: ./buildci.sh installdeps + - name: Configure gdc + run: ./buildci.sh configure + - name: Build gdc + run: ./buildci.sh build + - name: Run testsuite + run: ${{ contains(matrix.job_name, 'testsuite') && './buildci.sh testsuite' || 'echo disabled' }} + - name: Run unittests + run: ${{ contains(matrix.job_name, 'unittests') && './buildci.sh unittests' || 'echo disabled' }} diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000000..6e04e495e86f --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,48 @@ +version: v1.0 +name: GDC +agent: + machine: + type: e1-standard-2 + os_image: ubuntu1804 +execution_time_limit: + hours: 3 + +blocks: + - name: "Ubuntu 18.04" + dependencies: [] + execution_time_limit: + hours: 2 + task: + jobs: + - name: "Testsuite" + commands: + - checkout + - ./buildci.sh setup + - ./buildci.sh build + - ./buildci.sh testsuite + - name: "Unit Tests" + commands: + - checkout + - ./buildci.sh setup + - ./buildci.sh build + - ./buildci.sh unittests + - name: "OSX 10.15" + dependencies: [] + task: + agent: + machine: + type: a1-standard-4 + os_image: macos-xcode11 + jobs: + - name: "Testsuite" + commands: + - checkout + - ./buildci.sh setup + - ./buildci.sh build + - ./buildci.sh testsuite + - name: "Unit Tests" + commands: + - checkout + - ./buildci.sh setup + - ./buildci.sh build + - ./buildci.sh unittests diff --git a/README.md b/README.md new file mode 100644 index 000000000000..f9a40f3c86cb --- /dev/null +++ b/README.md @@ -0,0 +1,224 @@ +## The GDC D Compiler +[![Buildkite](https://img.shields.io/buildkite/58fd9d7cf59f6c774888051edb0e037fad6d97bcf04e53ac4f/ci/mainline.svg?style=flat&logo=dependabot&label=buildkite)](https://buildkite.com/d-programming-gdc/gcc) +[![Cirrus CI](https://img.shields.io/cirrus/github/D-Programming-GDC/gcc/ci/mainline?label=Cirrus%20CI&logo=Cirrus%20CI)](https://cirrus-ci.com/github/D-Programming-GDC/gcc/ci/mainline) +[![Bugzilla Issues](https://img.shields.io/badge/issues-Bugzilla-green.svg?style=flat)](https://gcc.gnu.org/bugzilla/buglist.cgi?component=d&list_id=299901&product=gcc&resolution=---) +[![License](https://img.shields.io/badge/license-GPLv2%2B-green)](https://github.com/D-Programming-GDC/gcc/blob/ci/mainline/COPYING) + +GDC is the GCC-based [D language][dlang] compiler, integrating the open source [DMDFE D][dmd] front end +with [GCC][gcc] as the backend. The GNU D Compiler (GDC) project was originally started by David Friedman +in 2004 until early 2007 when he disappeared from the D scene, and was no longer able to maintain GDC. +Following a revival attempt in 2008, GDC is now under the lead of Iain Buclaw who has been steering the +project since 2009 with the assistance of its contributors, without them the project would not have been +nearly as successful as it has been. + +Documentation on GDC is available from [the wiki][wiki]. Any bugs or issues found with using GDC should +be reported at [the GCC bugzilla site][bugs] with the bug component set to `d`. For help with GDC, the +[D.gnu][maillist] mailing list is the place to go with questions or problems. There's also a GDC IRC +channel at #d.gdc on FreeNode. Any questions which are related to the D language, but not directly to +the GDC compiler, can be asked in the [D forums][dforum]. You can find more information about D, including +example code, API documentation, tutorials and these forums at the [main D website][dlang]. + +### Building GDC + +Stable GDC releases for production usage should be obtained by downloading stable GCC sources +from the [GCC downloads][gcc-download] site. +For the latest experimental development version, simply download a [GCC snapshot][gcc-snapshot] or +checkout the GCC Git repository. Most GDC development directly targets the GCC Git repository, +so the latest GDC version is always available in the GCC Git. +Do not use the `ci/mainline` branch in this repository, as it is rebased regularly and contains exclusively +CI related changes. + +During certain development phases (e.g. when GCC is in a feature freeze) larger GDC changes may be staged +to the `devel/gdc` branch. This branch is rebased irregularly, do not rely on the commit ids to be +stable. + +If you need to clone this repo for some reason, you may want to do a shallow clone using the +`--depth 1 --no-single-branch` git options, as this repository is large. To compile GDC, add `--enable-languages=d` to the GCC configure flags and [start building][gdc-build]. + +### Using GDC + +Usage information can be found at ... + +### Contributing to GDC + +Starting with GCC 9.0.0, GDC has been merged into upstream GCC and all GDC development now follows the usual +GCC development process. Changes to GDC and related code can therefore be submitted +to the [gcc-patches mailing list][patches-ml] for review. + +It is possible to directly post patches to the [mailing list][patches-ml] and not to use this repository at all. +We however recommend using this repository to make use of the CI checks and the github review workflow. + +#### Submitting Changes + +To submit changes to GDC, simply fork this repository, create a new feature branch based on the `ci/mainline` branch, +then open a pull request against the **mainline** branch. We recommend using full clones for development, allthough +using shallow clones should also be possible. In code: + +```bash +# Initial one time setup: +# For repository on github, then clone your fork +git clone git@github.com:[you]/gcc.git +cd gcc +# Add the gdc repository as a remote +git remote add gdc git@github.com:D-Programming-GDC/gcc.git + +# Do this for every patch: +# Fetch latest upstream changes +git remote update +# Base a new branch on gdc/mainline +git checkout gdc/ci/mainline +git checkout -b pr12345 +# Make changes, commit +git commit [...] +git push origin pr12345:pr12345 +# Open a pull request on github, target branch: mainline +``` +Opening a pull request will automatically trigger our CI and test your changes on various machines. + +#### Changelogs +The GCC project requires keeping changes in the `Changelog` files. GCC ships a script which can generate +Changelog templates for us if we feed it a diff: +```bash +git diff gdc/ci/mainline | ./contrib/mklog +``` +*Note:* The above command generates the diff between `gdc/ci/mainline` and your local branch. If `gdc/ci/mainline` was +updated and you did a `git remote update`, `gdc/mainline` may have changes which are not yet in your branch. +In that case, rebase onto `gdc/mainline` first. + +The command outputs something like this: +``` +ChangeLog: + +2019-02-03 Johannes Pfau + + * test.d: New file. + +gcc/d/ChangeLog: + +2019-02-03 Johannes Pfau + + * dfile.txt: New file. + +libphobos/ChangeLog: + +2019-02-03 Johannes Pfau + + * phobosfile.txt: New file. + +``` + +The `ChangeLog:`, `libphobos/ChangeLog:` part gives the file into which the following changes need to be added. +Complete the changelog text and use the existing entries in the files for reference or see +the [GCC][changelog-doc] and [GNU][changelog-doc2] documentation. Also make sure to adhere to the line length limit of 80 characters. Then make the changelog somehow available for review: +Either commit the files, or preferable, just copy and paste the edited text output of `mklog` into your +pull request description. + + +### Getting Changes Into GCC Git + +After changes have been reviewed on github, they have to be pushed into the GCC Git. Pull requests will +not get merged into this repository. The following steps can be handled by GDC maintainers, although it is +possible to perform these by yourself as well. + +##### Sumbitting to the gcc-patches Mailing List + +Once the review and CI have passed on the github pull request page, the changes need to be submitted to the +`gcc-patches` mailing list. This can easily be done using [git send-email][git-send-email]: + +1. You might want to squash the commits. Each commit will become one email/patch so it might make sense + to combine commits here. +2. The changelog should preferrably be pasted into the email text, so do not include + commits modifying the changelog files. +3. If you had to regenerate any autogenerated files (e.g. configure from configure.ac) + you may keep these changes out of the patch for simplified review. The generated files + should still be present in the changelog. + +You'll have to configure `git send-email` once after you checked out the repository: +```bash +git config sendemail.to gcc-patches@gcc.gnu.org +``` +If you never used `git send-email` before, you'll also have to setup the SMTP settings once. +See [here][git-send-email] for details. + +Now to send the patches: +```bash +# Check which commits will be sent: +git log gdc/ci/mainline.. +# Check the complete diff which will be sent: +git diff gdc/ci/mainline.. +# Dry run to verify everything again +git send-email gdc/ci/mainline --annotate --dry-run +# Send the patches +git send-email gdc/ci/mainline --annotate +``` + +If you send multiple patches and want to write an introduction email, use the `--compose` argument for +`git send-email`. You can also generate patch files like this: +```bash +git format-patch gdc/ci/mainline.. +# Edit the *.patch files, add messages etc. +# Now send the patches +git send-email *.patch --dry-run +git send-email *.patch +``` + +##### Pushing Changes to Git + +This section is only relevant for GDC maintainers with GCC Git write access. There are certain rules when +pushing to Git, usually you're only allowed to push **after** the patches have been reviewed on the mailing list. +Refer to the [GCC documentation][gcc-git] for details. + +### Repository Information + +This repository is a fork of the [GCC git mirror][gcc-github]. + +#### Directory Structure + +All code branches contain the complete GCC tree. D sources are in `gcc/d` for the compiler +and in `libphobos` for the runtime library. Changes to files in `gcc/d/dmd` or `libphobos` +should be submitted to the [upstream dlang repositories][dlang-github] first if possible. +Refer to [gcc/d/README.gcc][gcc-d-readme] for more details. + +#### Branches + +Branches in this repository are organized in the following way: + +* CI branches: The `ci/mainline` branch and release branches `ci/gcc-*` are based on the same + branches in the upstream GCC git repository. The only changes compared to the upstream branches + are CI-related setup commits. CI branches are updated automatically to be kept in sync with + upstream and are rebased onto the upstream changes. These branches are effectively readonly: + We never merge into the branches in this repository. The CI related changes make it possible + to run CI based tests for any PR based on these branches, which is their sole purpose. +* The `devel/gdc` branch: If GCC is in a late [development stage][gcc-stage] this branch can accumulate + changes for the GCC release after the next one. It is essentially used to allow periodic merges from + [upstream DMD][dlang-github] when GCC development is frozen. Changes in the GCC `mainline` branch + are manually merged into this branch. When GCC enters stage 1 development again, this branch will be + rebased and pushed to upstream `mainline`. After that, the branch in this repository will be **rebased** + to mainline. +* Backport branches: The `gcc-*-bp` branches contain D frontend and library feature updates for released GCC versions. + Regression fixes should target the main `gcc-*-branch` branches instead, according to GCC rules. + + + +[home]: https://gdcproject.org +[dlang]: https://dlang.org +[gcc]: https://gcc.gnu.org +[dforum]: https://forum.dlang.org +[dmd]: https://github.com/dlang/dmd +[wiki]: https://wiki.dlang.org/GDC +[bugs]: https://gcc.gnu.org/bugzilla +[maillist]: https://forum.dlang.org/group/D.gnu +[email]: mailto:ibuclaw@gdcproject.org +[gcc-devel]: https://gcc.gnu.org/git/?p=gcc.git;a=shortlog +[patches-ml]: https://gcc.gnu.org/lists.html +[gcc-github]: https://github.com/gcc-mirror/gcc +[gcc-git]: https://gcc.gnu.org/gitwrite.html +[gcc-stage]: https://www.gnu.org/software/gcc/develop.html +[dlang-github]: https://github.com/dlang +[gdc-build]: https://wiki.dlang.org/GDC/Installation/Generic +[changelog-doc]: https://www.gnu.org/software/gcc/codingconventions.html#ChangeLogs +[changelog-doc2]: https://www.gnu.org/prep/standards/standards.html#Change-Logs +[git-send-email]: https://www.freedesktop.org/wiki/Software/PulseAudio/HowToUseGitSendEmail/ +[gcc-download]: https://www.gnu.org/software/gcc/releases.html +[gcc-d-readme]: https://github.com/D-Programming-GDC/gcc/blob/ci/mainline/gcc/d/README.gcc +[gcc-snapshot]: https://www.gnu.org/software/gcc/snapshots.html diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000000..8b9c1f5e72cf --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,48 @@ +jobs: +- job: Windows + timeoutInMinutes: 120 + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + x64: + OS: win64 + MODEL: 64 + ARCH: x64 + x86: + OS: win32 + MODEL: 32 + ARCH: x86 + steps: + - checkout: self + fetchDepth: 1 + - script: | + git clone https://github.com/lazka/msys2-ci-base.git %CD:~0,2%\msys64 + %CD:~0,2%\msys64\usr\bin\rm -rf %CD:~0,2%\msys64\.git + displayName: Install MSYS2 + - script: | + set PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem + %CD:~0,2%\msys64\usr\bin\pacman --noconfirm -Syyuu + displayName: Update MSYS2 + +- job: Linux_X86_ML + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-16.04' + variables: + AZURE: 'true' + steps: + - checkout: self + fetchDepth: 1 + - script: | + ./buildci.sh setup + displayName: Install prerequisites + - script: | + ./buildci.sh build + displayName: Building GCC + - script: | + ./buildci.sh testsuite + displayName: Running testsuite + - script: | + ./buildci.sh unittests + displayName: Running unit tests diff --git a/buildci.sh b/buildci.sh new file mode 100755 index 000000000000..1a37d4956fcd --- /dev/null +++ b/buildci.sh @@ -0,0 +1,542 @@ +#!/bin/bash +# This script is intended to be ran on platform on GitHub Action +# Other CI platforms in this script are legacy and only kept around to keep +# the configuration parts of it modularish. +# +# Following environmental variables are assume to be exported on GitHub Actions. +# +# - RUNNER_TOOL_CACHE +# - RUNNER_OS +# - GCC_CI_TARGET +# - GCC_CI_BOOTSTRAP +# +# See https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables +# +## Top-level build system configration. +gcc_prereqs="gmp-6.1.0.tar.bz2 mpfr-3.1.6.tar.bz2 mpc-1.0.3.tar.gz isl-0.18.tar.bz2" +host_package="9" + +export CC="gcc-${host_package}" +export CXX="g++-${host_package}" +export GDC="gdc-${host_package}" +export MAKE="make" + +environment() { + ## Determine what flags to use for configure, build and testing the compiler. + ## Commonize CI environment variables. + # + # project_dir: directory of checked out sources. + # cache_dir: tarballs of downloaded dependencies cached + # between builds. + # log_dir: where to save testsuite logs to upload them + # build_host: host triplet that build is ran from. + # build_host_canonical: canonical version of host triplet. + # build_target: target triplet of the compiler to build. + # build_target_canonical: canonical version of target triplet. + # make_flags: flags to pass to make. + # build_bootstrap: whether to enable bootstrap build. + # + if [ "${SEMAPHORE}" = "true" ]; then + if [ -z "${SEMAPHORE_CACHE_DIR}" ]; then + export SEMAPHORE_CACHE_DIR="$PWD/gcc-deps" + fi; + if [ -z "${SEMAPHORE_PROJECT_DIR}" ]; then + export SEMAPHORE_PROJECT_DIR="$PWD" + fi; + project_dir=${SEMAPHORE_PROJECT_DIR} + cache_dir=${SEMAPHORE_CACHE_DIR} + if [ "${SEMAPHORE_AGENT_MACHINE_OS_IMAGE}" != "" ]; then + cache restore $SEMAPHORE_PROJECT_NAME-$SEMAPHORE_AGENT_MACHINE_OS_IMAGE-deps + else + cache restore $SEMAPHORE_PROJECT_NAME-deps + fi + log_dir=${PWD}/logs + mkdir -p ${log_dir} + build_host=$($CC -dumpmachine) + build_host_canonical=$(${project_dir}/config.sub ${build_host}) + build_target=${build_host} + build_target_canonical=${build_host_canonical} + if [ "${SEMAPHORE_AGENT_MACHINE_OS_IMAGE}" = "macos-xcode11" ]; then + make_flags="-j$(sysctl -n hw.logicalcpu)" + else + make_flags="-j$(nproc)" + fi + build_bootstrap="disable" + elif [ "${BUILDKITE}" = "true" ]; then + project_dir=${PWD} + cache_dir=${BUILDKITE_CACHE_DIR} + log_dir=${PWD}/logs + mkdir -p ${log_dir} + if [ "${BUILDKITE_OS}" = "netbsd" ]; then + export CC="gcc" + export CXX="g++" + export GDC="gdc" + export MAKE="gmake" + make_flags="-j$(sysctl -n hw.ncpu)" + elif [ "${BUILDKITE_OS}" = "openbsd" ]; then + export CC="egcc" + export CXX="eg++" + export GDC="egdc" + export MAKE="gmake" + make_flags="-j$(sysctl -n hw.ncpu)" + elif [ "${BUILDKITE_OS}" = "alpine" ]; then + export CC="gcc" + export CXX="g++" + export GDC="gdc" + make_flags="-j$(nproc)" + else + make_flags="-j$(nproc)" + fi + build_host=$($CC -dumpmachine) + build_host_canonical=$(${project_dir}/config.sub ${build_host}) + build_target=${BUILDKITE_TARGET} + build_target_canonical=$(${project_dir}/config.sub ${build_target}) + make_flags="${make_flags} -sw LIBTOOLFLAGS=--silent" + build_bootstrap=${BUILDKITE_BOOTSTRAP} + elif [ "${CIRRUS_CI}" = "true" ]; then + project_dir=${PWD} + cache_dir="${PWD}/gcc-deps" + log_dir=${PWD}/logs + mkdir -p ${log_dir} + if [ "${CIRRUS_OS}" = "freebsd" ]; then + export CC="gcc${host_package}" + export CXX="g++${host_package}" + export GDC="gdc${host_package}" + export MAKE="gmake" + make_flags="-j$(sysctl -n hw.ncpu)" + elif [ "${CIRRUS_OS}" = "darwin" ]; then + make_flags="-j$(sysctl -n hw.logicalcpu)" + else + make_flags="-j$(nproc)" + fi + build_host=$($CC -dumpmachine) + build_host_canonical=$(${project_dir}/config.sub ${build_host}) + build_target=${build_host} + build_target_canonical=${build_host_canonical} + build_bootstrap="disable" + elif [ "${AZURE}" = "true" ]; then + project_dir=${PWD} + cache_dir="${PWD}/gcc-deps" + build_host=$($CC -dumpmachine) + build_host_canonical=$(${project_dir}/config.sub ${build_host}) + build_target=${build_host} + build_target_canonical=${build_host_canonical} + make_flags="-j$(nproc)" + build_bootstrap="disable" + elif [ "${CI}" = "true" ]; then + gcc_prereqs="" + project_dir=${PWD} + cache_dir=${RUNNER_TOOL_CACHE} + log_dir=${PWD}/logs + mkdir -p ${log_dir} + make_flags="-j$(nproc)" + build_host=$($CC -dumpmachine) + build_host_canonical=$(${project_dir}/config.sub ${build_host}) + build_target=${GCC_CI_TARGET} + build_target_canonical=$(${project_dir}/config.sub ${build_target}) + build_bootstrap=${GCC_CI_BOOTSTRAP} + else + echo "Unhandled CI environment" + exit 1 + fi + + ## Options determined by target, what steps to skip, or extra flags to add. + ## Also, should the testsuite be ran under a simulator? + # + # build_supports_phobos: whether to build phobos and run unittests. + # build_target_phobos: where to run the phobos testsuite from. + # build_enable_languages: which languages to build, this affects whether C++ + # or LTO tests are ran in the testsuite. + # build_prebuild_script: script to run after sources have been extracted. + # build_configure_flags: extra configure flags for the target. + # build_test_flags: options to pass to RUNTESTFLAGS. + # + build_supports_phobos='yes' + build_target_phobos='' + build_enable_languages='c++,d,lto' + build_prebuild_script='' + build_configure_flags='' + build_test_flags='' + + # Check whether this is a cross or multiarch compiler. + if [ "${build_host_canonical}" != "${build_target_canonical}" ]; then + multilib_targets=( $(${CC} -print-multi-lib | cut -f2 -d\;) ) + is_cross_compiler=1 + + for multilib in ${multilib_targets[@]}; do + build_multiarch=$(${CC} -print-multiarch ${multilib/@/-}) + build_multiarch_canonical=$(${project_dir}/config.sub ${build_multiarch}) + + # This is a multiarch compiler, update target to the host compiler. + if [ "${build_multiarch_canonical}" = "${build_target_canonical}" ]; then + build_target=$build_host + build_target_canonical=$build_host_canonical + build_target_phobos="${build_target}/$(${CC} ${multilib/@/-} -print-multi-directory)/libphobos" + build_test_flags="--target_board=unix{${multilib/@/-}}" + build_configure_flags='--enable-multilib --enable-multiarch' + is_cross_compiler=0 + break + fi + done + + # Building a cross compiler, need to explicitly say where to find native headers. + if [ ${is_cross_compiler} -eq 1 ]; then + build_configure_flags="--with-native-system-header-dir=/usr/${build_target}/include" + + # Note: setting target board to something other than "generic" only makes + # sense if phobos is being built. Without phobos, all runnable tests will + # all fail as being 'UNRESOLVED', and so are never ran anyway. + case ${build_target_canonical} in + arm*-*-*) + build_test_flags='--target_board=buildci-arm-sim' + ;; + *) + build_test_flags='--target_board=buildci-generic-sim' + ;; + esac + fi + fi + + if [ "${build_target_phobos}" = "" ]; then + build_target_phobos="${build_target}/libphobos" + fi + + # Unless requested, don't build with multilib. + if [ `expr "${build_configure_flags}" : '.*enable-multilib'` -eq 0 ]; then + build_configure_flags="--disable-multilib ${build_configure_flags}" + fi + + # If bootstrapping, be sure to turn off slow tree checking. + if [ "${build_bootstrap}" = "enable" ]; then + build_configure_flags="${build_configure_flags} \ + --enable-bootstrap --enable-checking=release" + else + build_configure_flags="${build_configure_flags} \ + --disable-bootstrap --enable-checking" + fi + + # Determine correct flags for configuring a compiler for target. + case ${build_target_canonical} in + arm-*-*eabihf) + build_configure_flags="${build_configure_flags} \ + --with-arch=armv7-a --with-fpu=vfpv3-d16 --with-float=hard --with-mode=thumb" + if [ `expr "${build_configure_flags}" : '.*enable-multilib'` -eq 1 ]; then + build_prebuild_script="${cache_dir}/patches/arm-multilib.sh" + fi + ;; + arm*-*-*eabi) + build_configure_flags="${build_configure_flags} \ + --with-arch=armv5t --with-float=soft" + ;; + mips-*-*|mipsel-*-*) + build_configure_flags="${build_configure_flags} \ + --with-arch=mips32r2" + ;; + mips64*-*-*) + build_configure_flags="${build_configure_flags} \ + --with-arch-64=mips64r2 --with-abi=64" + ;; + powerpc64le-*-*) + build_configure_flags="${build_configure_flags} \ + --with-cpu=power8 --with-long-double-128" + ;; + powerpc64-*-*) + build_configure_flags="${build_configure_flags} \ + --with-cpu=power7" + ;; + x86_64-*-darwin19) + build_configure_flags="${build_configure_flags} \ + --with-sysroot=/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk" + ;; + x86_64-*-openbsd*) + build_configure_flags="${build_configure_flags} \ + --with-gmp=/usr/local --enable-threads=posix --enable-default-pie" + build_prebuild_script="${cache_dir}/patches/openbsd-support.sh" + ;; + x86_64-*-*) + ;; + *) + build_supports_phobos='no' + build_enable_languages='c++,d --disable-lto' + ;; + esac + + if [ "${build_supports_phobos}" = "yes" ]; then + build_configure_flags="--enable-libphobos ${build_configure_flags}" + fi +} + +installdeps() { + ## Install build dependencies. + # Would save 1 minute if these were preinstalled in some docker image. + # But the network speed is nothing to complain about so far... + if [ "${SEMAPHORE}" = "true" ]; then + if [ "${SEMAPHORE_AGENT_MACHINE_OS_IMAGE}" = "ubuntu1804" ]; then + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -qq + sudo apt-get install -qq gcc-${host_package} g++-${host_package} gdc-${host_package} \ + autogen autoconf automake bison dejagnu flex patch || exit 1 + elif [ "${SEMAPHORE_AGENT_MACHINE_OS_IMAGE}" = "macos-xcode11" ]; then + brew update + brew install gcc@${host_package} autogen deja-gnu || exit 1 + elif [ "${SEMAPHORE_PLATFORM}" = "bionic-kvm" ]; then + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -qq + sudo apt-get install -qq gcc-${host_package} g++-${host_package} gdc-${host_package} \ + autogen autoconf automake bison dejagnu flex patch || exit 1 + else + echo "Unhandled CI environment" + exit 1 + fi + elif [ "${CIRRUS_CI}" = "true" ]; then + if [ "${CIRRUS_OS}" = "linux" ]; then + apt-get update -qq + apt-get install -qq software-properties-common + add-apt-repository -y ppa:ubuntu-toolchain-r/test + apt-get update -qq + apt-get install -qq gcc-${host_package} g++-${host_package} gdc-${host_package} \ + autogen autoconf automake bison curl dejagnu flex make patch || exit 1 + elif [ "${CIRRUS_OS}" = "freebsd" ]; then + pkg install -y gcc${host_package} \ + autogen autoconf automake autotools bison coreutils curl dejagnu flex gmake patch || exit 1 + elif [ "${CIRRUS_OS}" = "darwin" ]; then + brew update-reset + brew install gcc@${host_package} autogen deja-gnu xz || exit 1 + else + echo "Unhandled CI environment" + exit 1 + fi + elif [ "${CI}" = "true" ]; then + if [ "${RUNNER_OS}" = "Linux" ]; then + case ${GCC_CI_TARGET} in + arm*-*-*eabi) + install_packages="binutils-arm-linux-gnueabi libc6-dev-armel-cross" + ;; + arm*-*-*eabihf) + install_packages="binutils-arm-linux-gnueabihf libc6-dev-armhf-cross" + ;; + aarch64-*-*) + install_packages="binutils-aarch64-linux-gnu libc6-dev-arm64-cross" + ;; + mips-*-*) + install_packages="binutils-mips-linux-gnu libc6-dev-mips-cross" + ;; + mips64el-*-*) + install_packages="binutils-mips64el-linux-gnuabi64 libc6-dev-mips64el-cross" + ;; + mipsel-*-*) + install_packages="binutils-mipsel-linux-gnu libc6-dev-mipsel-cross" + ;; + powerpc64le-*-*) + install_packages="binutils-powerpc64le-linux-gnu libc6-dev-ppc64el-cross" + ;; + s390x-*-*) + install_packages="binutils-s390x-linux-gnu libc6-dev-s390x-cross" + ;; + sparc64-*-*) + install_packages="binutils-sparc64-linux-gnu libc6-dev-sparc64-cross" + ;; + *) + install_packages="binutils libc6-dev" + ;; + esac + sudo apt-get update -qq + sudo apt-get install -qq gcc-${host_package} g++-${host_package} gdc-${host_package} \ + ${install_packages} autogen autoconf automake dejagnu patch \ + libcurl4-gnutls-dev libgmp-dev libisl-dev libmpc-dev libmpfr-dev || exit 1 + else + echo "Unhandled CI environment" + exit 1 + fi + else + echo "Unhandled CI environment" + exit 1 + fi +} + +configure() { + ## And download GCC prerequisites. + # Makes use of local cache to save downloading on every build run. + for prereq in ${gcc_prereqs}; do + if [ ! -e ${cache_dir}/infrastructure/${prereq} ]; then + curl "http://gcc.gnu.org/pub/gcc/infrastructure/${prereq}" \ + --location --retry 50 --create-dirs -o ${cache_dir}/infrastructure/${prereq} || exit 1 + fi + tar -C ${project_dir} -xf ${cache_dir}/infrastructure/${prereq} + ln -s "${project_dir}/${prereq%.tar*}" "${project_dir}/${prereq%-*}" + done + + if [ "${SEMAPHORE}" = "true" ]; then + if [ "${SEMAPHORE_AGENT_MACHINE_OS_IMAGE}" != "" ]; then + cache store $SEMAPHORE_PROJECT_NAME-$SEMAPHORE_AGENT_MACHINE_OS_IMAGE-deps $cache_dir + else + cache store $SEMAPHORE_PROJECT_NAME-deps $cache_dir + fi + fi + + ## Apply any ad-hoc fixes to the sources. + if [ "${build_prebuild_script}" != "" ]; then + source ${build_prebuild_script} + fi + + ## Create the build directory. + # Build typically takes around 10 minutes with -j4, could this be cached across CI runs? + mkdir ${project_dir}/build + cd ${project_dir}/build + + ## Configure GCC to build a D compiler. + ${project_dir}/configure --prefix=/usr --libdir=/usr/lib --libexecdir=/usr/lib --with-sysroot=/ \ + --enable-languages=${build_enable_languages} --enable-link-mutex \ + --disable-werror --disable-libgomp --disable-libmudflap \ + --disable-libquadmath --disable-libitm --disable-libsanitizer \ + --build=${build_host} --host=${build_host} --target=${build_target} \ + ${build_configure_flags} --with-bugurl="http://bugzilla.gdcproject.org" +} + +setup() { + installdeps + environment + configure +} + +build() { + if [ "${build_bootstrap}" = "enable" ]; then + ## Build the entire project to completion. + cd ${project_dir}/build + ${MAKE} ${make_flags} + else + ## Build the bare-minimum in order to run tests. + cd ${project_dir}/build + ${MAKE} ${make_flags} all-gcc || exit 1 + + # Note: libstdc++ and libphobos are built separately so that build errors don't mix. + if [ "${build_supports_phobos}" = "yes" ]; then + ${MAKE} ${make_flags} all-target-libstdc++-v3 || exit 1 + ${MAKE} ${make_flags} all-target-libphobos || exit 1 + fi + fi +} + +testsuite() { + ## Run just the compiler testsuite. + cd ${project_dir}/build + + ${MAKE} check-gcc RUNTESTFLAGS="help.exp" + ${MAKE} ${make_flags} check-gcc-d RUNTESTFLAGS="${build_test_flags}" + + # Upload testsuite results + save_logs + + # For now, be lenient towards any failures, just report on them. + summary +} + +unittests() { + ## Run just the library unittests. + if [ "${build_supports_phobos}" = "yes" ]; then + cd ${project_dir}/build + if ! ${MAKE} ${make_flags} -C ${build_target_phobos} check RUNTESTFLAGS="${build_test_flags}"; then + save_logs + echo "== Unittest has failures ==" + exit 1 + fi + save_logs + fi +} + +summary() { + ## Processes *.{sum,log} files, producing a summary of all testsuite runs. + cd ${project_dir}/build + files=`find . -name \*.sum -print | sort` + anyfile=false + + for file in $files; do + if [ -f $file ]; then + anyfile=true + fi + done + + # Based on GCC testsuite summary scripts. + if [ "${anyfile}" = "true" ]; then + # We use cat instead of listing the files as arguments to AWK because + # GNU awk 3.0.0 would break if any of the filenames contained `=' and + # was preceded by an invalid variable name. + ( echo @TOPLEVEL_CONFIGURE_ARGUMENTS@ | ./config.status --file=-; cat $files ) | + awk ' + BEGIN { + lang=""; configflags = ""; + version="gcc"; + } + NR == 1 { + configflags = $0 " "; + srcdir = configflags; + sub(/\/configure\047? .*/, "", srcdir); + sub(/^\047/, "", srcdir); + if ( system("test -f " srcdir "/LAST_UPDATED") == 0 ) { + printf "LAST_UPDATED: "; + system("tail -1 " srcdir "/LAST_UPDATED"); + print ""; + } + + sub(/^[^ ]*\/configure\047? */, " ", configflags); + sub(/,;t t $/, " ", configflags); + sub(/ --with-gcc-version-trigger=[^ ]* /, " ", configflags); + sub(/ --norecursion /, " ", configflags); + sub(/ $/, "", configflags); + sub(/^ *$/, " none", configflags); + configflags = "configure flags:" configflags; + } + /^Running target / { print; } + /^Target / { if (host != "") next; else host = $3; } + /^Host / && host ~ /^unix\{.*\}$/ { host = $3 " " substr(host, 5); } + /^Native / { if (host != "") next; else host = $4; } + /^[ ]*=== [^ ]+ tests ===/ { + if (lang == "") lang = " "$2" "; else lang = " "; + } + $2 == "version" { + save = $0; $1 = ""; $2 = ""; version = $0; gsub(/^ */, "", version); gsub(/\r$/, "", version); $0 = save; + } + /\===.*Summary/ || /tests ===/ { print ""; print; blanks=1; } + /^(Target|Host|Native)/ { print; } + /^(XPASS|FAIL|UNRESOLVED|WARNING|ERROR|# of )/ { sub ("\r", ""); print; } + /^using:/ { print ""; print; print ""; } + /^$/ && blanks>0 { print; --blanks; } + END { + if (lang != "") { + print ""; + print "Compiler version: " prefix version lang; + print "Platform: " host; + print configflags; + } + } + { next; } + ' | sed "s/\([\`\$\\\\]\)/\\\\\\1/g" + fi +} + +save_logs() { + cd ${project_dir}/build + + test -e ./gcc/testsuite/gdc/gdc.log && mkdir -p ${log_dir}/gdc && \ + mv gcc/testsuite/gdc/*.{sum,log} ${log_dir}/gdc + test -e ./gcc/testsuite/gcc/gcc.log && mkdir -p ${log_dir}/gcc && \ + mv gcc/testsuite/gcc/*.{sum,log} ${log_dir}/gcc + test -e ./${build_target}/libphobos/testsuite/libphobos.log && mkdir -p ${log_dir}/libphobos && \ + mv ${build_target}/libphobos/testsuite/*.{sum,log} ${log_dir}/libphobos + + # Compress logs + find ${log_dir} \( -name \*.sum -o -name \*.log \) -exec xz \{\} \; +} + +## Run a single build task or all at once. +if [ "$1" != "" ]; then + # Skip calling environment if running setup, as dependencies might not be installed yet. + if [ "$1" != "setup" ]; then + environment + fi + $1 +else + setup + build + unittests +fi