Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

✨ Use custom-built Bazel #62

Closed
wants to merge 1 commit into from
Closed

✨ Use custom-built Bazel #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions baze_ll/baze_ll.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{ pkgs }:

# We use this customized variant of Bazel because some subtools in the regular
# variants have issues finding dynamic libstdc++ during runtime. We don't expect
# this to get fixed anytime soon, so we build a customized variant that
# statically links libc++ into the bazel sub-executables. Note that this doesn't
# make Bazel independent of libstdc++ as we still need it for openjdk.

# Overriding this version of Bazel to link against musl libc is possible but
# quite hacky and incredibly compile-time intensive. Since we need glibc anyways
# for heterogeneous targets it's currently not worth the extra effort to also
# support musl-based containers.

# TODO: Upstream fixes to nixpkgs so that we can support musl properly.

let
# As of Bazel 6.1.1 the abseil dependency in upb is so ancient that we can't
# build it with clang 15. We can still use libcxx 15 though.
LLVMOverride = {
stdenv = pkgs.overrideCC pkgs.llvmPackages_14.libcxxStdenv
(pkgs.wrapCCWith {
cc = pkgs.llvmPackages_14.clang-unwrapped;
bintools = pkgs.wrapBintoolsWith {
bintools = pkgs.llvmPackages_14.bintools-unwrapped;
};
});
};

# These libraries are statically linked into the final Bazel executable in
# place of libstdc++.
# TODO: This current implementation seems to work but still links libgcc_s
# which shouldn't be required.
libunwindStatic = pkgs.llvmPackages_15.libunwind.override {
enableShared = false;
};
libcxxabiStatic = pkgs.llvmPackages_15.libcxxabi.override {
libunwind = libunwindStatic;
enableShared = false;
};
libcxxStatic = pkgs.llvmPackages_15.libcxx.override {
libcxxabi = libcxxabiStatic;
enableShared = false;
};
compiler-rt = pkgs.llvmPackages_15.compiler-rt.override {
libcxxabi = libcxxabiStatic;
};

LLVMOverrideCXX = LLVMOverride // { libcxx = libcxxStatic; };

# Disable (most) transitive dependencies on libstdc++ (except for openjdk).
coreutils = pkgs.coreutils.override { gmpSupport = false; };
findutils = pkgs.findutils.override { inherit coreutils; };

env = {
NIX_CFLAGS_COMPILE = pkgs.lib.strings.concatStringsSep " " [
"-O3"
"-nostdinc++"
"-isystem${libcxxStatic.dev}/include/c++/v1"
"-isystem${libcxxabiStatic.dev}/include/c++/v1"
"-resource-dir=${pkgs.llvmPackages_14.clang}/resource-root"
];
NIX_LDFLAGS = pkgs.lib.strings.concatStringsSep " " [
# TODO: The current llvm stdenv doesn't use lld properly.
# Test the flags below if that changes.
# "-fuse-ld=lld"
# "--stdlib=libc++"
# "-rtlib=compiler-rt"
"-unwindlib=libunwind"
"-L${libcxxStatic}/lib"
"-lc++"
"-L${libcxxabiStatic}/lib"
"-lc++abi"
"-L${compiler-rt}/lib/linux"
"-L${libunwindStatic}/lib"
];
};
in
(pkgs.bazel_6.overrideAttrs (final: prev: {
inherit env;
})).override (LLVMOverrideCXX // {
inherit coreutils findutils;

# TODO: This is the default, but we need to change it ASAP since this is the
# only remaining edge to libstdc++. It's surprisingly tricky to build the jdk
# with an llvm toolchain, but it's surely possible.
buildJdk = pkgs.jdk11_headless;
})
2 changes: 1 addition & 1 deletion examples/flake.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

186 changes: 126 additions & 60 deletions flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,137 @@
fi
'';

baze_ll = import ./baze_ll/baze_ll.nix { inherit pkgs; };

bazel = { unfree ? false }: pkgs.writeShellScriptBin "bazel" (''
# Add the nix cflags and ldflags to the Bazel action envs.
# This is safe to do since the Nix environment is reproducible.
LL_NIX_CFLAGS_COMPILE=-isystem${pkgs.glibc.dev}/include:-isystem${pkgs.libxcrypt}/include
LL_NIX_LDFLAGS=-L${pkgs.glibc}/lib:-L${pkgs.libxcrypt}/lib

# These environment variables may be modified from outside of
# the bazel invocation.
LL_CFLAGS=''${LL_CFLAGS+$LL_CFLAGS:}$LL_NIX_CFLAGS_COMPILE
LL_LDFLAGS=''${LL_LDFLAGS+$LL_LDFLAGS:}$LL_NIX_LDFLAGS

# This must always be the linker from the glibc we compile
# and link against.
LL_DYNAMIC_LINKER=${pkgs.glibc}/lib/ld-linux-x86-64.so.2

# Flags for AMD dependencies.
LL_AMD_INCLUDES=-isystem${pkgs.libdrm.dev}/include:-isystem${pkgs.libdrm.dev}/include/libdrm:-isystem${pkgs.elfutils.dev}/include:-isystem${pkgs.numactl}/include:-isystem${pkgs.libglvnd.dev}/include:-isystem${pkgs.xorg.libX11.dev}/include:-isystem${pkgs.xorg.xorgproto}/include
LL_AMD_LIBRARIES=-L${pkgs.libdrm}/lib:-L${pkgs.numactl}/lib:-L=${pkgs.libglvnd}/lib:-L${pkgs.elfutils.out}/lib:-L${pkgs.libglvnd}/lib:-L${pkgs.xorg.libX11}/lib
LL_AMD_RPATHS=-rpath=${pkgs.libdrm}/lib:-rpath=${pkgs.numactl}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.elfutils.out}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.xorg.libX11}/lib

'' + (if unfree then ''
# Flags for CUDA dependencies.
LL_CUDA_TOOLKIT=${pkgsUnfree.cudaPackages_12.cudatoolkit}
LL_CUDA_RUNTIME=${pkgsUnfree.cudaPackages_12.cudatoolkit.lib}
LL_CUDA_DRIVER=${pkgsUnfree.linuxPackages_6_1.nvidia_x11}
'' else "") + ''

# Only used by rules_cc
BAZEL_CXXOPTS="-std=c++17:-O3:-nostdinc++:-nostdlib++:-isystem${pkgs.llvmPackages_15.libcxx.dev}/include/c++/v1"
BAZEL_LINKOPTS="-L${pkgs.llvmPackages_15.libcxx}/lib:-L${pkgs.llvmPackages_15.libcxxabi}/lib:-lc++:-Wl,-rpath,${pkgs.llvmPackages_15.libcxx}/lib,-rpath,${pkgs.llvmPackages_15.libcxxabi}/lib"

if [[
"$1" == "build" ||
"$1" == "coverage" ||
"$1" == "run" ||
"$1" == "test"
]]; then
${baze_ll}/bin/bazel $1 \
--action_env=LL_CFLAGS=$LL_CFLAGS \
--action_env=LL_LDFLAGS=$LL_LDFLAGS \
--action_env=LL_DYNAMIC_LINKER=$LL_DYNAMIC_LINKER \
--action_env=LL_AMD_INCLUDES=$LL_AMD_INCLUDES \
--action_env=LL_AMD_LIBRARIES=$LL_AMD_LIBRARIES \
--action_env=LL_AMD_RPATHS=$LL_AMD_RPATHS \
--action_env=LL_CUDA_TOOLKIT=$LL_CUDA_TOOLKIT \
--action_env=LL_CUDA_RUNTIME=$LL_CUDA_RUNTIME \
--action_env=LL_CUDA_DRIVER=$LL_CUDA_DRIVER \
--action_env=BAZEL_CXXOPTS=$BAZEL_CXXOPTS \
--action_env=BAZEL_LINKOPTS=$BAZEL_LINKOPTS \
''${@:2}
else
${baze_ll}/bin/bazel $@
fi
'');

image =
{ ll_env ? [
# Set custom LL_CFLAGS and LL_LDFALAGS here using nix template
# expansion and they will be made available via the external
# dependency mechanism.
"LL_CFLAGS="
"LL_LDFLAGS="
]
}: pkgs.dockerTools.buildLayeredImage {
name = "rules_ll_remote";
tag = "latest";

contents = [
# The default shell environment.
(bazel { unfree = false; })

# Minimal user setup. Required by Bazel.
pkgs.fakeNss

# Required for communication with trusted sources.
pkgs.cacert

# Toolchain for rules_cc.
pkgs.llvmPackages_15.clang
pkgs.llvmPackages_15.compiler-rt
pkgs.llvmPackages_15.libcxx
pkgs.llvmPackages_15.libcxxabi
pkgs.llvmPackages_15.libunwind
pkgs.llvmPackages_15.lld
pkgs.llvmPackages_15.stdenv

# Tools that we would usually forward from the host.
pkgs.bash
pkgs.coreutils
];

extraCommands = ''
mkdir rules_ll
cp -r ${./.}/* rules_ll

'';

config = {
Cmd = [ "ls" "/nix/store" ];
WorkingDir = "/rules_ll/examples";
Env = [
"CC=clang"
"LD=ld.lld"
"LD_LIBRARY_PATH=/lib"
] ++ ll_env;
};
};

hooks = import ./pre-commit-hooks.nix { inherit pkgs; };

in
rec {

hooks = import ./pre-commit-hooks.nix {
inherit pkgs;
packages = {
default = baze_ll;
test_image =
let openssl = (pkgs.openssl.override { static = true; }); in
image {
ll_env = [
"LL_CFLAGS=-I${openssl.dev}/include"
"LL_CFLAGS=-I${openssl.out}/lib"
];
};
};

checks = {
pre-commit-check = pre-commit-hooks-nix.lib.${system}.run {
src = ./.;
hooks = hooks;
inherit hooks;
};
};

Expand All @@ -83,68 +203,14 @@
inherit inputs pkgs;

modules = [{
pre-commit.hooks = hooks;
pre-commit = { inherit hooks; };

inherit env;

scripts.bazel.exec = (''
# Add the nix cflags and ldflags to the Bazel action envs.
# This is safe to do since the Nix environment is reproducible.
LL_NIX_CFLAGS_COMPILE=-isystem${pkgs.glibc.dev}/include:-isystem${pkgs.libxcrypt}/include
LL_NIX_LDFLAGS=-L${pkgs.glibc}/lib:-L${pkgs.libxcrypt}/lib

# These environment variables may be modified from outside of
# the bazel invocation.
LL_CFLAGS=''${LL_CFLAGS+$LL_CFLAGS:}$LL_NIX_CFLAGS_COMPILE
LL_LDFLAGS=''${LL_LDFLAGS+$LL_LDFLAGS:}$LL_NIX_LDFLAGS

# This must always be the linker from the glibc we compile
# and link against.
LL_DYNAMIC_LINKER=${pkgs.glibc}/lib/ld-linux-x86-64.so.2

# Flags for AMD dependencies.
LL_AMD_INCLUDES=-isystem${pkgs.libdrm.dev}/include:-isystem${pkgs.libdrm.dev}/include/libdrm:-isystem${pkgs.elfutils.dev}/include:-isystem${pkgs.numactl}/include:-isystem${pkgs.libglvnd.dev}/include:-isystem${pkgs.xorg.libX11.dev}/include:-isystem${pkgs.xorg.xorgproto}/include
LL_AMD_LIBRARIES=-L${pkgs.libdrm}/lib:-L${pkgs.numactl}/lib:-L=${pkgs.libglvnd}/lib:-L${pkgs.elfutils.out}/lib:-L${pkgs.libglvnd}/lib:-L${pkgs.xorg.libX11}/lib
LL_AMD_RPATHS=-rpath=${pkgs.libdrm}/lib:-rpath=${pkgs.numactl}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.elfutils.out}/lib:-rpath=${pkgs.libglvnd}/lib:-rpath=${pkgs.xorg.libX11}/lib

'' + (if unfree then ''
# Flags for CUDA dependencies.
LL_CUDA_TOOLKIT=${pkgsUnfree.cudaPackages_12.cudatoolkit}
LL_CUDA_RUNTIME=${pkgsUnfree.cudaPackages_12.cudatoolkit.lib}
LL_CUDA_DRIVER=${pkgsUnfree.linuxPackages_6_1.nvidia_x11}
'' else "") + ''

# Only used by rules_cc
BAZEL_CXXOPTS="-std=c++17:-O3:-nostdinc++:-nostdlib++:-isystem${pkgs.llvmPackages_15.libcxx.dev}/include/c++/v1"
BAZEL_LINKOPTS="-L${pkgs.llvmPackages_15.libcxx}/lib:-L${pkgs.llvmPackages_15.libcxxabi}/lib:-lc++:-Wl,-rpath,${pkgs.llvmPackages_15.libcxx}/lib,-rpath,${pkgs.llvmPackages_15.libcxxabi}/lib"

if [[
"$1" == "build" ||
"$1" == "coverage" ||
"$1" == "run" ||
"$1" == "test"
]]; then
bazelisk $1 \
--action_env=LL_CFLAGS=$LL_CFLAGS \
--action_env=LL_LDFLAGS=$LL_LDFLAGS \
--action_env=LL_DYNAMIC_LINKER=$LL_DYNAMIC_LINKER \
--action_env=LL_AMD_INCLUDES=$LL_AMD_INCLUDES \
--action_env=LL_AMD_LIBRARIES=$LL_AMD_LIBRARIES \
--action_env=LL_AMD_RPATHS=$LL_AMD_RPATHS \
--action_env=LL_CUDA_TOOLKIT=$LL_CUDA_TOOLKIT \
--action_env=LL_CUDA_RUNTIME=$LL_CUDA_RUNTIME \
--action_env=LL_CUDA_DRIVER=$LL_CUDA_DRIVER \
--action_env=BAZEL_CXXOPTS=$BAZEL_CXXOPTS \
--action_env=BAZEL_LINKOPTS=$BAZEL_LINKOPTS \
''${@:2}
else
bazelisk $@
fi
'');

packages = [
# Host toolchain.
pkgs.bazelisk
# pkgs.bazelisk
(bazel { inherit unfree; })
pkgs.llvmPackages_15.clang
pkgs.llvmPackages_15.compiler-rt
pkgs.llvmPackages_15.libcxx
Expand Down