Merge branch 'main' of https://github.com/compiler-explorer/misc-builder

 into main

.github/workflows/build.yml

@@ -7,7 +7,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        image: ['misc', 'rocm', 'compcert', 'rust-cg-gcc', 'rust-linux', 'iwyu', 'lc3', 'nasm', 'hylo', 'heaptrack', 'mads']
+        image:
+        - compcert
+        - heaptrack
+        - hylo
+        - iwyu
+        - lc3
+        - misc
+        - nasm
+        - pythran
+        - rocm
+        - rust-cg-gcc
+        - rust-linux
+        - vast
+        - mads
     steps:
     - name: Docker Setup Buildx
       uses: docker/setup-buildx-action@v2
@@ -18,7 +31,7 @@ jobs:
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - name: Build and push to Docker Hub
       id: hub_build
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         push: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
         tags: compilerexplorer/${{ matrix.image }}-builder:latest

Dockerfile.hylo

@@ -37,8 +37,8 @@ RUN mkdir /root/.ssh \
 RUN git clone https://github.com/compiler-explorer/infra /opt/compiler-explorer/infra
 
 RUN cd /opt/compiler-explorer/infra && make ce
-RUN /opt/compiler-explorer/infra/bin/ce_install install 'swift 5.8'
-ENV PATH="${PATH}:/opt/compiler-explorer/swift-5.8/usr/bin"
+RUN /opt/compiler-explorer/infra/bin/ce_install install 'swift 5.9'
+ENV PATH="${PATH}:/opt/compiler-explorer/swift-5.9/usr/bin"
 
 RUN mkdir -p /root
 COPY hylo /root/

Dockerfile.misc

@@ -44,7 +44,8 @@ RUN apt update -y -q && apt upgrade -y -q && apt update -y -q && \
     re2c \
     perl \
     cpanminus \
-    openssh-client 
+    openssh-client \
+    libgmp-dev
 
 RUN cd /tmp && \
     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \

Dockerfile.pythran

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get --assume-yes update \
+    && DEBIAN_FRONTEND=noninteractive apt-get --assume-yes \
+    --no-install-recommends install \
+        ca-certificates \
+        curl \
+        git \
+        bc \
+        make \
+        xz-utils
+
+
+RUN mkdir -p /root
+
+RUN mkdir -p /root/miniconda3
+RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /root/miniconda3/miniconda.sh
+RUN bash /root/miniconda3/miniconda.sh -b -u -p /root/miniconda3
+RUN rm -rf /root/miniconda3/miniconda.sh
+
+ENV PATH="/root/miniconda3/bin:${PATH}"
+
+COPY pythran/ /root/
+COPY common.sh /root/
+
+WORKDIR /root

Dockerfile.vast

@@ -0,0 +1,36 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y -q && apt upgrade -y -q
+RUN apt-get install -y -q \
+    build-essential \
+    curl \
+    git \
+    patchelf \
+    lld \
+    ninja-build \
+    python3-pip \
+    ssh \
+    software-properties-common
+
+RUN curl -sL https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz \
+    | tar zx -C /usr --strip-components=1
+
+RUN bash -c "$(curl -s -o - https://apt.llvm.org/llvm.sh)" llvm.sh 17
+
+RUN apt-get install -y -q \
+ libstdc++-12-dev \
+ llvm-17 \
+ libmlir-17 \
+ libmlir-17-dev \
+ mlir-17-tools \
+ libclang-17-dev
+
+RUN pip3 install lit
+
+RUN mkdir -p /root
+COPY vast /root/
+COPY common.sh /root/
+
+WORKDIR /root

hylo/build.sh

@@ -33,7 +33,7 @@ fi
 initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
 
 rm -rf "hylo-${VERSION}"
-git clone -q --depth 1 --single-branch -b "${BRANCH}" "${URL}" "hylo-${VERSION}"
+git clone -q --depth 1 --single-branch --recursive -b "${BRANCH}" "${URL}" "hylo-${VERSION}"
 
 cd "hylo-${VERSION}"
 swift package resolve

misc/build-cppfront.sh

@@ -54,14 +54,15 @@ STAGING_DIR=/opt/compiler-explorer/cppfront-${VERSION}
 GXXPATH=/opt/compiler-explorer/gcc-12.1.0
 
 rm -rf "${STAGING_DIR}"
-mkdir -p "${STAGING_DIR}"
+mkdir -p "${STAGING_DIR}/include"
 
 rm -rf "${SUBDIR}"
 git clone -q --depth 1 --single-branch -b "${BRANCH}" "${URL}" "${SUBDIR}"
 
 cd "${SUBDIR}"
 
 "${GXXPATH}/bin/g++" -O2 -std=c++20 -static -o "${STAGING_DIR}/cppfront" source/cppfront.cpp
+cp include/cpp2util.h "${STAGING_DIR}/include"
 
 export XZ_DEFAULTS="-T 0"
 tar Jcf "${OUTPUT}" --transform "s,^./,./${SUBDIR}/," -C "${STAGING_DIR}" .

misc/build-wyrm-transpiler.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -ex
+source common.sh
+
+VERSION=$1
+if [[ "${VERSION}" = "trunk" ]]; then
+    VERSION=trunk-$(date +%Y%m%d)
+    BRANCH=main
+else
+    BRANCH=V${VERSION}
+fi
+
+URL=https://github.com/jeremy-rifkin/wyrm.git
+
+FULLNAME=wyrm-${VERSION}.tar.xz
+OUTPUT=$2/${FULLNAME}
+
+REVISION="wyrm-$(get_remote_revision "${URL}" "heads/${BRANCH}")"
+LAST_REVISION="${3:-}"
+
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
+
+DIR=$(pwd)/wyrm
+STAGING_DIR=/opt/compiler-explorer/wyrm-${VERSION}
+
+git clone "${URL}" "${DIR}"
+
+cd "${DIR}/transpiler"
+
+mkdir build
+cd build
+export CXX=/opt/compiler-explorer/gcc-12.1.0/bin/g++
+export CC=/opt/compiler-explorer/gcc-12.1.0/bin/gcc
+cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$STAGING_DIR"
+ninja install
+
+cp -v libplugin.so "${STAGING_DIR}"
+
+patchelf --set-rpath '$ORIGIN/lib:/opt/compiler-explorer/gcc-12.1.0/lib64/' "${STAGING_DIR}/libplugin.so"
+
+complete "${STAGING_DIR}" "wyrm-${VERSION}" "${OUTPUT}"

pythran/build.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -exu
+source common.sh
+
+VERSION="$1"
+
+ROOT=$(pwd)
+FULLNAME=pythran-${VERSION}.tar.xz
+OUTPUT=${ROOT}/${FULLNAME}
+
+if [[ -d "${2}" ]]; then
+   OUTPUT=$2/${FULLNAME}
+else
+   OUTPUT=${2-$OUTPUT}
+fi
+
+REVISION="heaptrack-${VERSION}"
+LAST_REVISION="${3:-}"
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
+
+STAGING_DIR="/opt/compiler-explorer/pythran/pythran-${VERSION}"
+
+conda init
+
+eval "$(conda shell.bash hook)"
+conda activate base
+
+conda install conda-build -y
+conda deactivate
+
+PACKAGES="gcc_linux-64 \
+    gxx_linux-64 \
+    libgfortran-ng \
+    libgfortran5 \
+    libgcc-ng \
+    libgomp \
+    libstdcxx-ng"
+
+pushd /root/scratch
+conda build .
+popd
+
+mkdir -p $(dirname "$STAGING_DIR")
+
+conda create -y -p "$STAGING_DIR"
+
+## This may need some fine tuning if pythran bumps its dep on gcc from 13.2 to
+## something different.
+
+for P in $PACKAGES; do
+    conda install -y --use-local "$P=13.2.0=external*" -p "$STAGING_DIR"
+done
+
+conda install -y -c conda-forge pythran="$VERSION" -p "$STAGING_DIR"
+
+complete "${STAGING_DIR}" "pythran-${VERSION}" "${OUTPUT}"

pythran/scratch/meta.yaml

@@ -0,0 +1,29 @@
+{% set version = "13.2.0" %}
+{% set build = 0 %}
+
+package:
+  name: gcc-dummies
+  version: {{ version }}
+
+build:
+  number: {{ build }}
+
+outputs:
+  - name: gcc_linux-64
+    string: external_{{ build }}
+  - name: gxx_linux-64
+    string: external_{{ build }}
+  - name: libgfortran-ng
+    string: external_{{ build }}
+  - name: libgfortran5
+    string: external_{{ build }}
+  - name: libgcc-ng
+    string: external_{{ build }}
+  - name: libgomp
+    string: external_{{ build }}
+  - name: libstdcxx-ng
+    string: external_{{ build }}
+
+about:
+  license: GPL-3.0-only
+  summary: Dummy package for external GCC compiler and libs.

vast/README.md

@@ -0,0 +1,3 @@
+### Builder for Vast
+
+This is the build setup for [VAST](https://github.com/trailofbits/vast). 

vast/build.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+## $1 : version
+## $2 : destination: a directory
+## $3 : last revision: a revision descriptor which may be fetched from the cache.
+
+set -exu
+source common.sh
+
+ROOT=$(pwd)
+VERSION=$1
+URL="https://github.com/trailofbits/vast"
+
+if echo "${VERSION}" | grep 'trunk'; then
+    VERSION=trunk-$(date +%Y%m%d)
+    BRANCH=master
+    REVISION=$(get_remote_revision "${URL}" "heads/${BRANCH}")
+else
+    BRANCH=${VERSION}
+    REVISION=$(get_remote_revision "${URL}" "tags/${BRANCH}")
+fi
+
+FULLNAME=vast-${VERSION}.tar.xz
+OUTPUT=${ROOT}/${FULLNAME}
+LAST_REVISION="${3:-}"
+
+if [[ -d "${2}" ]]; then
+   OUTPUT=$2/${FULLNAME}
+else
+   OUTPUT=${2-$OUTPUT}
+fi
+
+initialise "${REVISION}" "${OUTPUT}" "${LAST_REVISION}"
+
+STAGING_DIR=/opt/compiler-explorer/vast-${VERSION}
+
+rm -rf "vast-${VERSION}"
+git clone -q --depth 1 --recursive --single-branch -b "${BRANCH}" "${URL}" "vast-${VERSION}"
+
+cd "vast-${VERSION}"
+
+export CC=/usr/bin/clang-17
+export CXX=/usr/bin/clang++-17
+cmake --preset ninja-multi-default \
+  --toolchain ./cmake/lld.toolchain.cmake \
+  --install-prefix "${STAGING_DIR}" \
+  -DLLVM_EXTERNAL_LIT=/usr/local/bin/lit \
+  -DCMAKE_INSTALL_RPATH:PATH="\$ORIGIN/../lib" \
+  -DCMAKE_PREFIX_PATH="/usr/lib/llvm-17;/usr/lib/llvm-17/lib/cmake/mlir;/usr/lib/llvm-17/lib/cmake/clang"
+
+cmake --build --preset ninja-rel
+cmake --install ./builds/ninja-multi-default
+
+# Copy all shared object dependencies into the release directory to create a hermetic build, per
+# Compiler Explorer requirements. Update rpath for these objects to $ORIGIN.
+# To ensure we only muck with rpaths for these objects, do this work in a temporary directory.
+# This code copied and modified from compiler-explorer/cobol-builder/build/build.sh
+cp $(ldd "${STAGING_DIR}/bin/vast-front" | grep -E  '=> /' | grep -Ev 'lib(pthread|c|dl|rt).so' | awk '{print $3}') "${STAGING_DIR}/lib"
+patchelf --set-rpath '$ORIGIN/../lib' $(find ${STAGING_DIR}/lib/ -name \*.so\*)
+
+complete "${STAGING_DIR}" "vast-${VERSION}" "${OUTPUT}"