From ddeecce2821dc477e03bf426aa47467b6c557698 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 12 Nov 2024 11:21:08 +0000 Subject: [PATCH] chore(deps): implement Nix testing as separate derivation from env (#10473) Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- .github/workflows/nix.yml | 9 +++- flake.lock | 6 +-- flake.nix | 6 +++ nix/ibis.nix | 54 ---------------------- nix/overlay.nix | 95 ++++++++++++++++++++++++++------------- nix/tests.nix | 41 +++++++++++++++++ 6 files changed, 123 insertions(+), 88 deletions(-) delete mode 100644 nix/ibis.nix create mode 100644 nix/tests.nix diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 088a8c675b94..d03fc0177f1b 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -61,13 +61,20 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} extraPullNames: nix-community - - name: nix build and test + - name: nix build environment run: | set -euo pipefail version='${{ matrix.python-version }}' nix build ".#ibis${version//./}" --fallback --keep-going --print-build-logs + - name: nix test + run: | + set -euo pipefail + + version='${{ matrix.python-version }}' + nix build ".#ibis${version//./}.passthru.tests.pytest" --fallback --keep-going --print-build-logs + - name: nix build devShell # TODO: dev shell doesn't yet build on macos-14 (aarch64-darwin) continue-on-error: ${{ matrix.os == 'macos-14' }} diff --git a/flake.lock b/flake.lock index 1ea4df84e819..48aa81a2eb38 100644 --- a/flake.lock +++ b/flake.lock @@ -108,11 +108,11 @@ ] }, "locked": { - "lastModified": 1731223088, - "narHash": "sha256-qkij76/APgVXsgagQ0z0lTy3qeorfTuY5S91tzROSe0=", + "lastModified": 1731375416, + "narHash": "sha256-foYwh+QEgda8EuymxOydvkv5nVaVsPW7yN0/CLLDCM0=", "owner": "adisbladis", "repo": "uv2nix", - "rev": "638bb9e42147ebd6344053b20f05516b7b4674d7", + "rev": "3f1bfa56f3acd32762690a1b6362f75d30748cab", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 6d0e729646d0..8af712ef10a9 100644 --- a/flake.nix +++ b/flake.nix @@ -150,6 +150,12 @@ update-lock-files check-release-notes-spelling; }; + checks = { + ibis310-pytest = pkgs.ibis310.passthru.tests.pytest; + ibis311-pytest = pkgs.ibis311.passthru.tests.pytest; + ibis312-pytest = pkgs.ibis312.passthru.tests.pytest; + }; + devShells = rec { ibis310 = mkDevShell pkgs.ibisDevEnv310; ibis311 = mkDevShell pkgs.ibisDevEnv311; diff --git a/nix/ibis.nix b/nix/ibis.nix deleted file mode 100644 index 7359fe6e039c..000000000000 --- a/nix/ibis.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ pkgs, python, mkEnv, stdenv, ... }: -let - pythonEnv = mkEnv python { - editable = false; - deps = { - ibis-framework = [ - # Groups - "tests" - # Extras - "duckdb" - "datafusion" - "sqlite" - "polars" - "decompiler" - "visualization" - ]; - }; - }; - -in -stdenv.mkDerivation { - name = "ibis-framework-test"; - nativeCheckInputs = [ pythonEnv pkgs.graphviz-nox ]; - src = ../.; - doCheck = true; - preCheck = '' - set -euo pipefail - - ln -s ${pkgs.ibisTestingData} $PWD/ci/ibis-testing-data - - HOME="$(mktemp -d)" - export HOME - ''; - checkPhase = '' - runHook preCheck - pytest -m datafusion - pytest -m 'core or duckdb or sqlite or polars' --numprocesses $NIX_BUILD_CORES --dist loadgroup - runHook postCheck - ''; - - # Don't run the fixup phase(s), to avoid permissions errors - dontFixup = true; - - # ibis-framework was already built as a part of the env, this is just running - # tests. Symlink the built test env for convenience. - # - # Note: Testing could technically be done as a part of the virtualenv - # constructor derivation. - installPhase = '' - runHook preInstall - ln -s ${pythonEnv} $out - runHook postInstall - ''; -} diff --git a/nix/overlay.nix b/nix/overlay.nix index 18534519e579..3517c2414ab4 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -7,19 +7,43 @@ let sourcePreference = "wheel"; }; + # Create an overlay enabling editable mode for all local dependencies. + # This is for usage with `nix develop` editableOverlay = - # Create an overlay enabling editable mode for all local dependencies. - # This is for usage with nix-nix develop workspace.mkEditablePyprojectOverlay { root = "$REPO_ROOT"; }; + # Build fixups overlay pyprojectOverrides = import ./pyproject-overrides.nix { inherit pkgs; }; - mkEnv = python: { deps ? workspace.deps.all, editable ? true }: - # This devShell uses uv2nix to construct a virtual environment purely from Nix, using the same dependency specification as the application. - # - # This means that any changes done to your local files do not require a rebuild. + # Adds tests to ibis-framework.passthru.tests + testOverlay = import ./tests.nix { + inherit pkgs; + deps = defaultDeps; + }; + + # Default dependencies for env + defaultDeps = { + ibis-framework = [ + "duckdb" + "datafusion" + "sqlite" + "polars" + "decompiler" + "visualization" + ]; + }; + + mkEnv' = + { + # Python dependency specification + deps + , # Installs ibis-framework as an editable package for use with `nix develop`. + # This means that any changes done to your local files do not require a rebuild. + editable + , + }: python: let # Construct package set pythonSet = @@ -30,10 +54,29 @@ let (lib.composeManyExtensions ([ envOverlay pyprojectOverrides - ] ++ lib.optional editable editableOverlay)); + ] + ++ lib.optionals editable [ editableOverlay ] + ++ lib.optionals (!editable) [ testOverlay ])); in # Build virtual environment - pythonSet.mkVirtualEnv "ibis-${python.pythonVersion}" deps; + (pythonSet.mkVirtualEnv "ibis-${python.pythonVersion}" deps).overrideAttrs (_old: { + # Add passthru.tests from ibis-framework to venv passthru. + # This is used to build tests by CI. + passthru = { + inherit (pythonSet.ibis-framework.passthru) tests; + }; + }); + + mkEnv = mkEnv' { + deps = defaultDeps; + editable = false; + }; + + mkDevEnv = mkEnv' { + # Enable all dependencies for development shell + deps = workspace.deps.all; + editable = true; + }; inherit (pkgs) lib stdenv; in @@ -46,30 +89,22 @@ in sha256 = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU="; }; - ibis310 = pkgs.callPackage ./ibis.nix { - python = pkgs.python310; - inherit mkEnv; - }; - - ibis311 = pkgs.callPackage ./ibis.nix { - python = pkgs.python311; - inherit mkEnv; - }; - - ibis312 = pkgs.callPackage ./ibis.nix { - python = pkgs.python312; - inherit mkEnv; - }; + ibis310 = mkEnv pkgs.python310; + ibis311 = mkEnv pkgs.python311; + ibis312 = mkEnv pkgs.python312; - ibisDevEnv310 = mkEnv pkgs.python310 { }; - ibisDevEnv311 = mkEnv pkgs.python311 { }; - ibisDevEnv312 = mkEnv pkgs.python312 { }; + ibisDevEnv310 = mkDevEnv pkgs.python310; + ibisDevEnv311 = mkDevEnv pkgs.python311; + ibisDevEnv312 = mkDevEnv pkgs.python312; - ibisSmallDevEnv = mkEnv pkgs.python312 { - deps = { - ibis-framework = [ "dev" ]; - }; - }; + ibisSmallDevEnv = mkEnv' + { + deps = { + ibis-framework = [ "dev" ]; + }; + editable = false; + } + pkgs.python312; duckdb = super.duckdb.overrideAttrs ( _: lib.optionalAttrs (stdenv.isAarch64 && stdenv.isLinux) { diff --git a/nix/tests.nix b/nix/tests.nix new file mode 100644 index 000000000000..eaa8cbceeca5 --- /dev/null +++ b/nix/tests.nix @@ -0,0 +1,41 @@ +{ pkgs, deps }: +let + inherit (pkgs) stdenv; +in +final: prev: { + ibis-framework = prev.ibis-framework.overrideAttrs (old: { + passthru = old.passthru // { + tests = old.passthru.tests or { } // { + pytest = + let + pythonEnv = final.mkVirtualEnv "ibis-framework-test-env" (deps // { + # Use default dependencies from overlay.nix + enabled tests group. + ibis-framework = deps.ibis-framework or [ ] ++ [ "tests" ]; + }); + in + stdenv.mkDerivation { + name = "ibis-framework-test"; + nativeCheckInputs = [ pythonEnv pkgs.graphviz-nox ]; + src = ../.; + doCheck = true; + preCheck = '' + set -euo pipefail + + ln -s ${pkgs.ibisTestingData} $PWD/ci/ibis-testing-data + + HOME="$(mktemp -d)" + export HOME + ''; + checkPhase = '' + runHook preCheck + pytest -m datafusion + pytest -m 'core or duckdb or sqlite or polars' --numprocesses $NIX_BUILD_CORES --dist loadgroup + runHook postCheck + ''; + + installPhase = "mkdir $out"; + }; + }; + }; + }); +}