Merge branch 'master' into fix-jvm-coverage-remove-redundant

docs/Gemfile

@@ -1,4 +1,4 @@
 source "https://rubygems.org"
 gem 'github-pages', group: :jekyll_plugins
 
-gem "webrick", "~> 1.7"
+gem "webrick", "~> 1.8"

docs/Gemfile.lock

@@ -249,14 +249,14 @@ GEM
       unf_ext
     unf_ext (0.0.8.2)
     unicode-display_width (1.8.0)
-    webrick (1.7.0)
+    webrick (1.8.2)
 
 PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
   github-pages
-  webrick (~> 1.7)
+  webrick (~> 1.8)
 
 BUNDLED WITH
    2.2.33

infra/base-images/base-builder-jvm/Dockerfile

@@ -14,7 +14,7 @@
 #
 ################################################################################
 
-FROM gcr.io/oss-fuzz-base/base-builder
+FROM gcr.io/oss-fuzz-base/base-builder AS base
 
 ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
 ENV JAVA_15_HOME /usr/lib/jvm/java-15-openjdk-amd64
@@ -27,6 +27,7 @@ RUN install_java.sh
 
 RUN chmod 777 /usr/local/bin && chmod 777 /usr/local/lib
 
+FROM base AS builder
 RUN useradd -m jazzer_user
 
 USER jazzer_user
@@ -54,10 +55,12 @@ RUN cp $(bazel cquery --output=files //src/main/java/com/code_intelligence/jazze
     cp $(bazel cquery --output=files //deploy:jazzer-api) $JAZZER_API_PATH && \
     cp $(bazel cquery --output=files //deploy:jazzer-junit) $JAZZER_JUNIT_PATH
 
-USER root
+FROM base AS final
 
-RUN rm -rf /home/jazzer_user/.cache/bazel /home/jazzer_user/.cache/bazelisk && \
-    rm -rf $SRC/jazzer
+COPY --from=builder /usr/local/bin/jazzer_agent_deploy.jar /usr/local/bin/jazzer_agent_deploy.jar
+COPY --from=builder /usr/local/bin/jazzer_driver /usr/local/bin/jazzer_driver
+COPY --from=builder $JAZZER_API_PATH $JAZZER_API_PATH
+COPY --from=builder $JAZZER_JUNIT_PATH $JAZZER_JUNIT_PATH
 
 RUN chmod 755 /usr/local/bin && chmod 755 /usr/local/lib
 

infra/base-images/base-builder/Dockerfile

@@ -19,9 +19,9 @@ FROM gcr.io/oss-fuzz-base/base-clang
 COPY install_deps.sh /
 RUN /install_deps.sh && rm /install_deps.sh
 
-# Build and install latest Python 3 (3.8.3).
-ENV PYTHON_VERSION 3.8.3
-RUN export PYTHON_DEPS="\
+# Build and install latest Python 3.10.
+ENV PYTHON_VERSION 3.10.14
+RUN PYTHON_DEPS="\
         zlib1g-dev \
         libncurses5-dev \
         libgdbm-dev \
@@ -39,12 +39,14 @@ RUN export PYTHON_DEPS="\
     tar -xvf Python-$PYTHON_VERSION.tar.xz && \
     cd Python-$PYTHON_VERSION && \
     ./configure --enable-optimizations --enable-shared && \
-    make -j install && \
+    make -j$(nproc) install && \
     ldconfig && \
-    ln -s /usr/bin/python3 /usr/bin/python && \
+    ln -s /usr/local/bin/python3 /usr/local/bin/python && \
     cd .. && \
     rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION && \
-    rm -rf /usr/local/lib/python3.8/test && \
+    rm -rf /usr/local/lib/python${PYTHON_VERSION%.*}/test && \
+    python3 -m ensurepip && \
+    python3 -m pip install --upgrade pip && \
     apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888
 
 

infra/base-images/base-builder/compile_python_fuzzer

@@ -32,7 +32,7 @@ if [[ $SANITIZER = *introspector* ]]; then
     # we enter the virtual environment in the following lines because we need
     # to use the same python environment that installed the fuzzer dependencies.
     python3 /fuzz-introspector/frontends/python/prepare_fuzz_imports.py $fuzzer_path isossfuzz
-    
+
     # We must ensure python3.9, this is because we use certain
     # AST logic from there.
     # The below should probably be refined
@@ -84,7 +84,7 @@ then
   if [[ ! -d "/pysecsan" ]];
   then
     pushd /usr/local/lib/sanitizers/pysecsan
-    python3 setup.py install
+    python3 -m pip install .
     popd
   fi
 

infra/base-images/base-builder/install_python.sh

@@ -19,5 +19,5 @@ echo "ATHERIS INSTALL"
 unset CFLAGS CXXFLAGS
 # PYI_STATIC_ZLIB=1 is needed for installing pyinstaller 5.0
 export PYI_STATIC_ZLIB=1
-LIBFUZZER_LIB=$( echo /usr/local/lib/clang/*/lib/x86_64-unknown-linux-gnu/libclang_rt.fuzzer_no_main.a ) pip3 install -v --no-cache-dir "atheris>=2.1.1" "pyinstaller==5.0.1" "setuptools==42.0.2" "coverage==6.3.2"
+LIBFUZZER_LIB=$( echo /usr/local/lib/clang/*/lib/x86_64-unknown-linux-gnu/libclang_rt.fuzzer_no_main.a ) pip3 install -v --no-cache-dir "atheris>=2.3.0" "pyinstaller==6.10.0" "setuptools==72.1.0" "coverage==6.3.2"
 rm -rf /tmp/*

infra/base-images/base-builder/sanitizers/pysecsan/pysecsan/command_injection.py

@@ -98,7 +98,7 @@ def hook_pre_exec_os_system(cmd):
                             'Command injection')
 
 
-def hook_pre_exec_eval(cmd):
+def hook_pre_exec_eval(cmd, *args, **kwargs):
   """Hook for eval. Experimental atm."""
   res = check_code_injection_match(cmd, check_unquoted=True)
   if res is not None:

infra/base-images/base-builder/sanitizers/pysecsan/pysecsan/redos.py

@@ -72,7 +72,7 @@ def hook_post_exec_re_compile(retval, pattern, flags=None):
   sanlib.sanitizer_log('Inside of post compile hook', sanlib.LOG_DEBUG)
   wrapper_object = sanlib.create_object_wrapper(
       findall=(hook_pre_exec_re_pattern_findall,
-               hook_pre_exec_re_pattern_findall))
+               hook_post_exec_re_pattern_findall))
   hooked_object = wrapper_object(retval)
   return hooked_object
 

infra/base-images/base-builder/sanitizers/pysecsan/pysecsan/sanlib.py

@@ -22,7 +22,7 @@
 import functools
 import subprocess
 import traceback
-import importlib
+import importlib.util
 
 from typing import Any, Callable, Optional
 from pysecsan import command_injection, redos, yaml_deserialization
@@ -54,7 +54,7 @@ def sanitizer_log_always(msg, log_prefix=True):
 def is_module_present(mod_name):
   """Identify if module is importable."""
   # pylint: disable=deprecated-method
-  return importlib.find_loader(mod_name) is not None
+  return importlib.util.find_spec(mod_name) is not None
 
 
 def _log_bug(bug_title):

infra/base-images/base-runner/Dockerfile

@@ -25,6 +25,11 @@ RUN cargo install rustfilt
 FROM gcr.io/oss-fuzz-base/base-clang AS base-clang
 FROM gcr.io/oss-fuzz-base/base-builder-ruby AS base-ruby
 
+# The base builder image compiles a specific Python version. Using a multi-stage build
+# to copy that same Python interpreter into the runner image saves build time and keeps
+# the Python versions in sync.
+FROM gcr.io/oss-fuzz-base/base-builder AS base-builder
+
 # Real image that will be used later.
 FROM gcr.io/oss-fuzz-base/base-image
 
@@ -36,6 +41,18 @@ COPY --from=base-clang /usr/local/bin/llvm-cov \
      /usr/local/bin/llvm-symbolizer \
      /usr/local/bin/
 
+# Copy the pre-compiled Python binaries and libraries
+COPY --from=base-builder /usr/local/bin/python3.10 /usr/local/bin/python3.10
+COPY --from=base-builder /usr/local/lib/libpython3.10.so.1.0 /usr/local/lib/libpython3.10.so.1.0
+COPY --from=base-builder /usr/local/include/python3.10 /usr/local/include/python3.10
+COPY --from=base-builder /usr/local/lib/python3.10 /usr/local/lib/python3.10
+COPY --from=base-builder /usr/local/bin/pip3 /usr/local/bin/pip3
+
+# Create symbolic links to ensure compatibility
+RUN ldconfig && \
+    ln -s /usr/local/bin/python3.10 /usr/local/bin/python3 && \
+    ln -s /usr/local/bin/python3.10 /usr/local/bin/python
+
 COPY install_deps.sh /
 RUN /install_deps.sh && rm /install_deps.sh
 
@@ -46,8 +63,11 @@ RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage
     cd /opt/code_coverage && \
     git checkout edba4873b5e8a390e977a64c522db2df18a8b27d && \
     pip3 install wheel && \
+    # If version "Jinja2==2.10" is in requirements.txt, bump it to a patch version that
+    # supports upgrading its MarkupSafe dependency to a Python 3.10 compatible release:
+    sed -i 's/Jinja2==2.10/Jinja2==2.10.3/' requirements.txt && \
     pip3 install -r requirements.txt && \
-    pip3 install MarkupSafe==0.23 && \
+    pip3 install MarkupSafe==2.0.1 && \
     pip3 install coverage==6.3.2
 
 # Default environment options for various sanitizers.

infra/base-images/base-runner/install_deps.sh

@@ -20,12 +20,10 @@
 apt-get update && apt-get install -y \
     binutils \
     file \
+    ca-certificates \
     fonts-dejavu \
     git \
     libcap2 \
-    python3 \
-    python3-pip \
-    python3-setuptools \
     rsync \
     unzip \
     wget \

projects/aptos-core/Dockerfile

@@ -15,7 +15,7 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder-rust
-RUN apt-get update && apt-get install -y make autoconf automake libclang-dev libtool pkg-config
+RUN apt-get update && apt-get install -y make autoconf automake libclang-dev libtool pkg-config libudev-dev
 RUN git clone --depth=1 https://github.com/aptos-labs/aptos-core.git
 WORKDIR aptos-core
 COPY build.sh $SRC

projects/astroid/Dockerfile

@@ -18,19 +18,7 @@ FROM gcr.io/oss-fuzz-base/base-builder-python
 
 RUN pip3 install --disable-pip-version-check --no-cache-dir pip==24.2
 
-# Pin the astroid version to 3.2.4 until Python 3.9+ is supported on OSS-Fuzz:
-# https://github.com/google/oss-fuzz/pull/12027
-RUN git clone --branch=v3.2.4 https://github.com/pylint-dev/astroid
-
-# Apply crash fixes to fix coverage builds:
-# https://github.com/pylint-dev/astroid/pull/2596
-# https://github.com/pylint-dev/astroid/pull/2597
-RUN cd astroid && \
-    git show dfe1ccce8576fd5854e7eb578985d10f464c1b67 > crash.diff && \
-    git apply --exclude=ChangeLog crash.diff && \
-    git show d174ca24bde0bba7df63f024124381efe908330d > crash.diff && \
-    git apply crash.diff && \
-    rm crash.diff
+RUN git clone --depth=1 --no-tags https://github.com/pylint-dev/astroid
 
 COPY *.py *.sh "$SRC/"
 

projects/bitcoin-core/Dockerfile

@@ -26,12 +26,6 @@ RUN apt-get update && apt-get install -y \
 RUN git clone --depth=1 https://github.com/bitcoin/bitcoin.git bitcoin-core
 RUN git clone --depth=1 https://github.com/bitcoin-core/qa-assets bitcoin-core/assets && \
   rm -rf bitcoin-core/assets/.git  # Remove git history to save storage
-RUN git clone --depth 1 https://github.com/guidovranken/cryptofuzz
 RUN git clone --depth 1 https://github.com/bitcoin-core/secp256k1.git
-RUN git clone --depth 1 https://github.com/randombit/botan.git
-RUN git clone --depth 1 https://github.com/trezor/trezor-firmware.git
-RUN git clone --depth 1 https://github.com/google/wycheproof.git
-RUN wget https://archives.boost.io/release/1.84.0/source/boost_1_84_0.tar.bz2
 WORKDIR bitcoin-core
 COPY build.sh $SRC/
-COPY build_cryptofuzz.sh $SRC/

projects/bitcoin-core/build.sh

@@ -18,11 +18,6 @@
 # Print date to embed it into build logs
 date
 
-if [ "$SANITIZER" != "introspector" ]; then
-  # Temporarily skip this under introspector
-  $SRC/build_cryptofuzz.sh
-fi
-
 cd $SRC/bitcoin-core/
 
 # Build dependencies