From f1d0df27c94fbad199281ddedd8a2c1b021e1cba Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Thu, 6 Apr 2023 01:24:44 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Use=20custom-built=20Bazel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The variant with bazelisk didn't work well with nix-built multi-layer remote execution images. Switch to an overridden Bazel from nixpkgs and add logic to create such images. The images about 2.6GB in size which is almost as small as we can get for an image containing Bazel and a Clang/LLVM C++ toolchain. We might be able to shave off ~250MB by building python with that same toolchain instead of building it with GCC. Signed-off-by: Aaron Siddhartha Mondal --- baze_ll/baze_ll.nix | 87 +++++++++++++++++++++ examples/flake.lock | 2 +- flake.nix | 186 ++++++++++++++++++++++++++++++-------------- 3 files changed, 214 insertions(+), 61 deletions(-) create mode 100644 baze_ll/baze_ll.nix diff --git a/baze_ll/baze_ll.nix b/baze_ll/baze_ll.nix new file mode 100644 index 00000000..a9f61bc4 --- /dev/null +++ b/baze_ll/baze_ll.nix @@ -0,0 +1,87 @@ +{ pkgs }: + +# We use this customized variant of Bazel because some subtools in the regular +# variants have issues finding dynamic libstdc++ during runtime. We don't expect +# this to get fixed anytime soon, so we build a customized variant that +# statically links libc++ into the bazel sub-executables. Note that this doesn't +# make Bazel independent of libstdc++ as we still need it for openjdk. + +# Overriding this version of Bazel to link against musl libc is possible but +# quite hacky and incredibly compile-time intensive. Since we need glibc anyways +# for heterogeneous targets it's currently not worth the extra effort to also +# support musl-based containers. + +# TODO: Upstream fixes to nixpkgs so that we can support musl properly. + +let + # As of Bazel 6.1.1 the abseil dependency in upb is so ancient that we can't + # build it with clang 15. We can still use libcxx 15 though. + LLVMOverride = { + stdenv = pkgs.overrideCC pkgs.llvmPackages_14.libcxxStdenv + (pkgs.wrapCCWith { + cc = pkgs.llvmPackages_14.clang-unwrapped; + bintools = pkgs.wrapBintoolsWith { + bintools = pkgs.llvmPackages_14.bintools-unwrapped; + }; + }); + }; + + # These libraries are statically linked into the final Bazel executable in + # place of libstdc++. + # TODO: This current implementation seems to work but still links libgcc_s + # which shouldn't be required. + libunwindStatic = pkgs.llvmPackages_15.libunwind.override { + enableShared = false; + }; + libcxxabiStatic = pkgs.llvmPackages_15.libcxxabi.override { + libunwind = libunwindStatic; + enableShared = false; + }; + libcxxStatic = pkgs.llvmPackages_15.libcxx.override { + libcxxabi = libcxxabiStatic; + enableShared = false; + }; + compiler-rt = pkgs.llvmPackages_15.compiler-rt.override { + libcxxabi = libcxxabiStatic; + }; + + LLVMOverrideCXX = LLVMOverride // { libcxx = libcxxStatic; }; + + # Disable (most) transitive dependencies on libstdc++ (except for openjdk). + coreutils = pkgs.coreutils.override { gmpSupport = false; }; + findutils = pkgs.findutils.override { inherit coreutils; }; + + env = { + NIX_CFLAGS_COMPILE = pkgs.lib.strings.concatStringsSep " " [ + "-O3" + "-nostdinc++" + "-isystem${libcxxStatic.dev}/include/c++/v1" + "-isystem${libcxxabiStatic.dev}/include/c++/v1" + "-resource-dir=${pkgs.llvmPackages_14.clang}/resource-root" + ]; + NIX_LDFLAGS = pkgs.lib.strings.concatStringsSep " " [ + # TODO: The current llvm stdenv doesn't use lld properly. + # Test the flags below if that changes. + # "-fuse-ld=lld" + # "--stdlib=libc++" + # "-rtlib=compiler-rt" + "-unwindlib=libunwind" + "-L${libcxxStatic}/lib" + "-lc++" + "-L${libcxxabiStatic}/lib" + "-lc++abi" + "-L${compiler-rt}/lib/linux" + "-L${libunwindStatic}/lib" + ]; + }; +in +(pkgs.bazel_6.overrideAttrs (final: prev: { + inherit env; +})).override (LLVMOverrideCXX // { + inherit coreutils findutils; + + # TODO: This is the default, but we need to change it ASAP since this is the + # only remaining edge to libstdc++. It's surprisingly tricky to build the jdk + # with an llvm toolchain, but it's surely possible. + buildJdk = pkgs.jdk11_headless; +}) diff --git a/examples/flake.lock b/examples/flake.lock index 9134d1f0..f974354f 100644 --- a/examples/flake.lock +++ b/examples/flake.lock @@ -378,7 +378,7 @@ }, "locked": { "lastModified": 0, - "narHash": "sha256-Z47/D2KopE/Bv/HwjI5XDJNkZZqpE+xy9rRRLe1eeOo=", + "narHash": "sha256-p8JWe/wX5YBz7o50pfud6cB5z5IR7f+2rd5SjzwSxjE=", "path": "../", "type": "path" }, diff --git a/flake.nix b/flake.nix index 640e8d77..2e8b40ab 100644 --- a/flake.nix +++ b/flake.nix @@ -46,17 +46,137 @@ fi ''; + baze_ll = import ./baze_ll/baze_ll.nix { inherit pkgs; }; + + bazel = { unfree ? false }: pkgs.writeShellScriptBin "bazel" ('' + # Add the nix cflags and ldflags to the Bazel action envs. + # This is safe to do since the Nix environment is reproducible. + LL_NIX_CFLAGS_COMPILE=-isystem${pkgs.glibc.dev}/include:-isystem${pkgs.libxcrypt}/include + LL_NIX_LDFLAGS=-L${pkgs.glibc}/lib:-L${pkgs.libxcrypt}/lib + + # These environment variables may be modified from outside of + # the bazel invocation. + LL_CFLAGS=''${LL_CFLAGS+$LL_CFLAGS:}$LL_NIX_CFLAGS_COMPILE + LL_LDFLAGS=''${LL_LDFLAGS+$LL_LDFLAGS:}$LL_NIX_LDFLAGS + + # This must always be the linker from the glibc we compile + # and link against. + LL_DYNAMIC_LINKER=${pkgs.glibc}/lib/ld-linux-x86-64.so.2 + + # Flags for AMD dependencies. + LL_AMD_INCLUDES=-isystem${pkgs.libdrm.dev}/include:-isystem${pkgs.libdrm.dev}/include/libdrm:-isystem${pkgs.elfutils.dev}/include:-isystem${pkgs.numactl}/include:-isystem${pkgs.libglvnd.dev}/include:-isystem${pkgs.xorg.libX11.dev}/include:-isystem${pkgs.xorg.xorgproto}/include + LL_AMD_LIBRARIES=-L${pkgs.libdrm}/lib:-L${pkgs.numactl}/lib:-L=${pkgs.libglvnd}/lib:-L${pkgs.elfutils.out}/lib:-L${pkgs.libglvnd}/lib:-L${pkgs.xorg.libX11}/lib + LL_AMD_RPATHS=-rpath=${pkgs.libdrm}/lib:-rpath=${pkgs.numactl}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.elfutils.out}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.xorg.libX11}/lib + + '' + (if unfree then '' + # Flags for CUDA dependencies. + LL_CUDA_TOOLKIT=${pkgsUnfree.cudaPackages_12.cudatoolkit} + LL_CUDA_RUNTIME=${pkgsUnfree.cudaPackages_12.cudatoolkit.lib} + LL_CUDA_DRIVER=${pkgsUnfree.linuxPackages_6_1.nvidia_x11} + '' else "") + '' + + # Only used by rules_cc + BAZEL_CXXOPTS="-std=c++17:-O3:-nostdinc++:-nostdlib++:-isystem${pkgs.llvmPackages_15.libcxx.dev}/include/c++/v1" + BAZEL_LINKOPTS="-L${pkgs.llvmPackages_15.libcxx}/lib:-L${pkgs.llvmPackages_15.libcxxabi}/lib:-lc++:-Wl,-rpath,${pkgs.llvmPackages_15.libcxx}/lib,-rpath,${pkgs.llvmPackages_15.libcxxabi}/lib" + + if [[ + "$1" == "build" || + "$1" == "coverage" || + "$1" == "run" || + "$1" == "test" + ]]; then + ${baze_ll}/bin/bazel $1 \ + --action_env=LL_CFLAGS=$LL_CFLAGS \ + --action_env=LL_LDFLAGS=$LL_LDFLAGS \ + --action_env=LL_DYNAMIC_LINKER=$LL_DYNAMIC_LINKER \ + --action_env=LL_AMD_INCLUDES=$LL_AMD_INCLUDES \ + --action_env=LL_AMD_LIBRARIES=$LL_AMD_LIBRARIES \ + --action_env=LL_AMD_RPATHS=$LL_AMD_RPATHS \ + --action_env=LL_CUDA_TOOLKIT=$LL_CUDA_TOOLKIT \ + --action_env=LL_CUDA_RUNTIME=$LL_CUDA_RUNTIME \ + --action_env=LL_CUDA_DRIVER=$LL_CUDA_DRIVER \ + --action_env=BAZEL_CXXOPTS=$BAZEL_CXXOPTS \ + --action_env=BAZEL_LINKOPTS=$BAZEL_LINKOPTS \ + ''${@:2} + else + ${baze_ll}/bin/bazel $@ + fi + ''); + + image = + { ll_env ? [ + # Set custom LL_CFLAGS and LL_LDFALAGS here using nix template + # expansion and they will be made available via the external + # dependency mechanism. + "LL_CFLAGS=" + "LL_LDFLAGS=" + ] + }: pkgs.dockerTools.buildLayeredImage { + name = "rules_ll_remote"; + tag = "latest"; + + contents = [ + # The default shell environment. + (bazel { unfree = false; }) + + # Minimal user setup. Required by Bazel. + pkgs.fakeNss + + # Required for communication with trusted sources. + pkgs.cacert + + # Toolchain for rules_cc. + pkgs.llvmPackages_15.clang + pkgs.llvmPackages_15.compiler-rt + pkgs.llvmPackages_15.libcxx + pkgs.llvmPackages_15.libcxxabi + pkgs.llvmPackages_15.libunwind + pkgs.llvmPackages_15.lld + pkgs.llvmPackages_15.stdenv + + # Tools that we would usually forward from the host. + pkgs.bash + pkgs.coreutils + ]; + + extraCommands = '' + mkdir rules_ll + cp -r ${./.}/* rules_ll + + ''; + + config = { + Cmd = [ "ls" "/nix/store" ]; + WorkingDir = "/rules_ll/examples"; + Env = [ + "CC=clang" + "LD=ld.lld" + "LD_LIBRARY_PATH=/lib" + ] ++ ll_env; + }; + }; + + hooks = import ./pre-commit-hooks.nix { inherit pkgs; }; + in rec { - hooks = import ./pre-commit-hooks.nix { - inherit pkgs; + packages = { + default = baze_ll; + test_image = + let openssl = (pkgs.openssl.override { static = true; }); in + image { + ll_env = [ + "LL_CFLAGS=-I${openssl.dev}/include" + "LL_CFLAGS=-I${openssl.out}/lib" + ]; + }; }; checks = { pre-commit-check = pre-commit-hooks-nix.lib.${system}.run { src = ./.; - hooks = hooks; + inherit hooks; }; }; @@ -83,68 +203,14 @@ inherit inputs pkgs; modules = [{ - pre-commit.hooks = hooks; + pre-commit = { inherit hooks; }; inherit env; - scripts.bazel.exec = ('' - # Add the nix cflags and ldflags to the Bazel action envs. - # This is safe to do since the Nix environment is reproducible. - LL_NIX_CFLAGS_COMPILE=-isystem${pkgs.glibc.dev}/include:-isystem${pkgs.libxcrypt}/include - LL_NIX_LDFLAGS=-L${pkgs.glibc}/lib:-L${pkgs.libxcrypt}/lib - - # These environment variables may be modified from outside of - # the bazel invocation. - LL_CFLAGS=''${LL_CFLAGS+$LL_CFLAGS:}$LL_NIX_CFLAGS_COMPILE - LL_LDFLAGS=''${LL_LDFLAGS+$LL_LDFLAGS:}$LL_NIX_LDFLAGS - - # This must always be the linker from the glibc we compile - # and link against. - LL_DYNAMIC_LINKER=${pkgs.glibc}/lib/ld-linux-x86-64.so.2 - - # Flags for AMD dependencies. - LL_AMD_INCLUDES=-isystem${pkgs.libdrm.dev}/include:-isystem${pkgs.libdrm.dev}/include/libdrm:-isystem${pkgs.elfutils.dev}/include:-isystem${pkgs.numactl}/include:-isystem${pkgs.libglvnd.dev}/include:-isystem${pkgs.xorg.libX11.dev}/include:-isystem${pkgs.xorg.xorgproto}/include - LL_AMD_LIBRARIES=-L${pkgs.libdrm}/lib:-L${pkgs.numactl}/lib:-L=${pkgs.libglvnd}/lib:-L${pkgs.elfutils.out}/lib:-L${pkgs.libglvnd}/lib:-L${pkgs.xorg.libX11}/lib - LL_AMD_RPATHS=-rpath=${pkgs.libdrm}/lib:-rpath=${pkgs.numactl}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.elfutils.out}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.xorg.libX11}/lib - - '' + (if unfree then '' - # Flags for CUDA dependencies. - LL_CUDA_TOOLKIT=${pkgsUnfree.cudaPackages_12.cudatoolkit} - LL_CUDA_RUNTIME=${pkgsUnfree.cudaPackages_12.cudatoolkit.lib} - LL_CUDA_DRIVER=${pkgsUnfree.linuxPackages_6_1.nvidia_x11} - '' else "") + '' - - # Only used by rules_cc - BAZEL_CXXOPTS="-std=c++17:-O3:-nostdinc++:-nostdlib++:-isystem${pkgs.llvmPackages_15.libcxx.dev}/include/c++/v1" - BAZEL_LINKOPTS="-L${pkgs.llvmPackages_15.libcxx}/lib:-L${pkgs.llvmPackages_15.libcxxabi}/lib:-lc++:-Wl,-rpath,${pkgs.llvmPackages_15.libcxx}/lib,-rpath,${pkgs.llvmPackages_15.libcxxabi}/lib" - - if [[ - "$1" == "build" || - "$1" == "coverage" || - "$1" == "run" || - "$1" == "test" - ]]; then - bazelisk $1 \ - --action_env=LL_CFLAGS=$LL_CFLAGS \ - --action_env=LL_LDFLAGS=$LL_LDFLAGS \ - --action_env=LL_DYNAMIC_LINKER=$LL_DYNAMIC_LINKER \ - --action_env=LL_AMD_INCLUDES=$LL_AMD_INCLUDES \ - --action_env=LL_AMD_LIBRARIES=$LL_AMD_LIBRARIES \ - --action_env=LL_AMD_RPATHS=$LL_AMD_RPATHS \ - --action_env=LL_CUDA_TOOLKIT=$LL_CUDA_TOOLKIT \ - --action_env=LL_CUDA_RUNTIME=$LL_CUDA_RUNTIME \ - --action_env=LL_CUDA_DRIVER=$LL_CUDA_DRIVER \ - --action_env=BAZEL_CXXOPTS=$BAZEL_CXXOPTS \ - --action_env=BAZEL_LINKOPTS=$BAZEL_LINKOPTS \ - ''${@:2} - else - bazelisk $@ - fi - ''); - packages = [ # Host toolchain. - pkgs.bazelisk + # pkgs.bazelisk + (bazel { inherit unfree; }) pkgs.llvmPackages_15.clang pkgs.llvmPackages_15.compiler-rt pkgs.llvmPackages_15.libcxx