use debian bookworm in docker container (#1311)

This returns to an upgrade first attempted in:
  #1255
That upgrade ran into sandbox trouble, which eventually proved to
be a small change in the layout of directories in bookworm relative
to buster (`/lib64` became a symlink).

.github/workflows/docker_latest.yml

@@ -127,9 +127,36 @@ jobs:
       - name: Install Google Chrome for Testing
         run: ./test/test_env.sh node_modules/selenium-webdriver/bin/linux/selenium-manager
 
-      - name: Run tests
+      - name: Run tests with default settings
         if: ${{ !inputs.disable_tests }}
-        run: TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker
+        run: |
+          export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}
+          export VERBOSE=1
+          export DEBUG=1
+          export MOCHA_WEBDRIVER_HEADLESS=1
+          yarn run test:docker
+
+      - name: Run some tests with gvisor and python2
+        if: ${{ !inputs.disable_tests }}
+        run: |
+          export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}
+          export VERBOSE=1
+          export DEBUG=1
+          export MOCHA_WEBDRIVER_HEADLESS=1
+          export GREP_TESTS='should support basic editing'
+          export TEST_DOCKER_OPTIONS='-e GRIST_SANDBOX_FLAVOR=gvisor -e PYTHON_VERSION_ON_CREATION=2'
+          yarn run test:docker
+
+      - name: Run some tests with gvisor and python3
+        if: ${{ !inputs.disable_tests }}
+        run: |
+          export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}
+          export VERBOSE=1
+          export DEBUG=1
+          export MOCHA_WEBDRIVER_HEADLESS=1
+          export GREP_TESTS='should support basic editing'
+          export TEST_DOCKER_OPTIONS='-e GRIST_SANDBOX_FLAVOR=gvisor -e PYTHON_VERSION_ON_CREATION=3'
+          yarn run test:docker
 
       - name: Re-enable the ext/ directory
         if: ${{ !inputs.disable_tests }}

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch AS ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:18-buster AS builder
+FROM node:22-bookworm AS builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -46,7 +46,7 @@ RUN \
 ################################################################################
 
 # Fetch python3.11
-FROM python:3.11-slim-buster AS collector-py3
+FROM python:3.11-slim-bookworm AS collector-py3
 ADD sandbox/requirements3.txt requirements3.txt
 RUN \
   pip3 install -r requirements3.txt
@@ -59,15 +59,22 @@ RUN \
 # the workarounds needed to keep it are getting silly.
 # It doesn't exist in recent Debian, so we need to reach back
 # to buster.
+# Many Python2 imports require the ffi foreign-function interface
+# library binary, of course present on modern debian but with
+# a different ABI (currently version 8, versus version 6 for this
+# version of Python2). We move it from an achitecture-specific location
+# to a standard location so we can pick it up and copy it across later.
+# This will no longer be necessary when support for Python2 is dropped.
+# The Grist data engine code will not work without it.
 FROM debian:buster-slim AS collector-py2
 ADD sandbox/requirements.txt requirements.txt
 RUN \
   apt update && \
   apt install -y --no-install-recommends python2 python-pip python-setuptools \
   build-essential libxml2-dev libxslt-dev python-dev zlib1g-dev && \
   pip2 install wheel && \
-  pip2 install -r requirements.txt 
-
+  pip2 install -r requirements.txt && \
+  find /usr/lib -iname "libffi.so.6*" -exec cp {} /usr/local/lib \;
 
 ################################################################################
 ## Sandbox collection stage
@@ -76,16 +83,19 @@ RUN \
 # Fetch gvisor-based sandbox. Note, to enable it to run within default
 # unprivileged docker, layers of protection that require privilege have
 # been stripped away, see https://github.com/google/gvisor/issues/4371
-# The sandbox binary is built on buster, but remains compatible with recent
-# Debian.
+# The standalone sandbox binary is built on buster, but remains compatible
+# with recent Debian.
+# If you'd like to use unmodified gvisor, you should be able to just drop
+# in the standard runsc binary and run the container with any extra permissions
+# it needs.
 FROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox
 
 ################################################################################
 ## Run-time stage
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:18-buster-slim
+FROM node:22-bookworm-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.
@@ -163,6 +173,12 @@ WORKDIR /grist
 # settings, you can get sandboxing as follows:
 #   docker run --env GRIST_SANDBOX_FLAVOR=gvisor -p 8484:8484 -it <image>
 #
+# "NODE_OPTIONS=--no-deprecation" is set because there is a punycode
+# deprecation nag that is relevant to developers but not to users.
+# TODO: upgrade package.json to avoid using all package versions
+# using the punycode functionality that may be removed in future
+# versions of node.
+#
 ENV \
   PYTHON_VERSION_ON_CREATION=3 \
   GRIST_ORG_IN_PATH=true \
@@ -174,6 +190,7 @@ ENV \
   GRIST_SESSION_COOKIE=grist_core \
   GVISOR_FLAGS="-unprivileged -ignore-cgroups" \
   GRIST_SANDBOX_FLAVOR=unsandboxed \
+  NODE_OPTIONS="--no-deprecation" \
   TYPEORM_DATABASE=/persist/home.sqlite3
 
 EXPOSE 8484

sandbox/gvisor/run.py

@@ -161,7 +161,12 @@ def preserve(*locations, short_failure=False):
   preserve("/usr/bin")
 
 preserve("/usr/local/lib")
-if os.path.exists('/lib64'):
+
+# Do not attempt to include symlink directories, they are not supported
+# and will cause obscure failures. On debian bookworm /lib64 is a
+# symlink and we do not appear to need it, relative to debian buster
+# where it is a real directory.
+if os.path.exists('/lib64') and not os.path.islink('/lib64'):
   preserve("/lib64")
 if os.path.exists('/usr/lib64'):
   preserve("/usr/lib64")

test/test_under_docker.sh

@@ -37,6 +37,7 @@ if [[ "${DEBUG:-}" == 1 ]]; then
   GRIST_LOG_HTTP_BODY="true"
 fi
 
+set -x
 docker run --name $DOCKER_CONTAINER --rm \
   --env VERBOSE=${DEBUG:-} \
   -p $PORT:$PORT --env PORT=$PORT \
@@ -47,7 +48,9 @@ docker run --name $DOCKER_CONTAINER --rm \
   --env GRIST_LOG_HTTP_BODY=${GRIST_LOG_HTTP_BODY:-false} \
   --env TEST_SUPPORT_API_KEY=api_key_for_support \
   --env GRIST_TEMPLATE_ORG=templates \
+  ${TEST_DOCKER_OPTIONS:-} \
   ${TEST_IMAGE:-gristlabs/grist} &
+set +x
 
 DOCKER_PID="$!"