chore(deps): implement Nix testing as separate derivation from env (#…

…10473)

Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com>

.github/workflows/nix.yml

@@ -61,13 +61,20 @@ jobs:
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
           extraPullNames: nix-community
 
-      - name: nix build and test
+      - name: nix build environment
         run: |
           set -euo pipefail
 
           version='${{ matrix.python-version }}'
           nix build ".#ibis${version//./}" --fallback --keep-going --print-build-logs
 
+      - name: nix test
+        run: |
+          set -euo pipefail
+
+          version='${{ matrix.python-version }}'
+          nix build ".#ibis${version//./}.passthru.tests.pytest" --fallback --keep-going --print-build-logs
+
       - name: nix build devShell
         # TODO: dev shell doesn't yet build on macos-14 (aarch64-darwin)
         continue-on-error: ${{ matrix.os == 'macos-14' }}

flake.nix

@@ -150,6 +150,12 @@
             update-lock-files check-release-notes-spelling;
         };
 
+        checks = {
+          ibis310-pytest = pkgs.ibis310.passthru.tests.pytest;
+          ibis311-pytest = pkgs.ibis311.passthru.tests.pytest;
+          ibis312-pytest = pkgs.ibis312.passthru.tests.pytest;
+        };
+
         devShells = rec {
           ibis310 = mkDevShell pkgs.ibisDevEnv310;
           ibis311 = mkDevShell pkgs.ibisDevEnv311;

nix/overlay.nix

@@ -7,19 +7,43 @@ let
     sourcePreference = "wheel";
   };
 
+  # Create an overlay enabling editable mode for all local dependencies.
+  # This is for usage with `nix develop`
   editableOverlay =
-    # Create an overlay enabling editable mode for all local dependencies.
-    # This is for usage with nix-nix develop
     workspace.mkEditablePyprojectOverlay {
       root = "$REPO_ROOT";
     };
 
+  # Build fixups overlay
   pyprojectOverrides = import ./pyproject-overrides.nix { inherit pkgs; };
 
-  mkEnv = python: { deps ? workspace.deps.all, editable ? true }:
-    # This devShell uses uv2nix to construct a virtual environment purely from Nix, using the same dependency specification as the application.
-    #
-    # This means that any changes done to your local files do not require a rebuild.
+  # Adds tests to ibis-framework.passthru.tests
+  testOverlay = import ./tests.nix {
+    inherit pkgs;
+    deps = defaultDeps;
+  };
+
+  # Default dependencies for env
+  defaultDeps = {
+    ibis-framework = [
+      "duckdb"
+      "datafusion"
+      "sqlite"
+      "polars"
+      "decompiler"
+      "visualization"
+    ];
+  };
+
+  mkEnv' =
+    {
+      # Python dependency specification
+      deps
+    , # Installs ibis-framework as an editable package for use with `nix develop`.
+      # This means that any changes done to your local files do not require a rebuild.
+      editable
+    ,
+    }: python:
     let
       # Construct package set
       pythonSet =
@@ -30,10 +54,29 @@ let
           (lib.composeManyExtensions ([
             envOverlay
             pyprojectOverrides
-          ] ++ lib.optional editable editableOverlay));
+          ]
+          ++ lib.optionals editable [ editableOverlay ]
+          ++ lib.optionals (!editable) [ testOverlay ]));
     in
     # Build virtual environment
-    pythonSet.mkVirtualEnv "ibis-${python.pythonVersion}" deps;
+    (pythonSet.mkVirtualEnv "ibis-${python.pythonVersion}" deps).overrideAttrs (_old: {
+      # Add passthru.tests from ibis-framework to venv passthru.
+      # This is used to build tests by CI.
+      passthru = {
+        inherit (pythonSet.ibis-framework.passthru) tests;
+      };
+    });
+
+  mkEnv = mkEnv' {
+    deps = defaultDeps;
+    editable = false;
+  };
+
+  mkDevEnv = mkEnv' {
+    # Enable all dependencies for development shell
+    deps = workspace.deps.all;
+    editable = true;
+  };
 
   inherit (pkgs) lib stdenv;
 in
@@ -46,30 +89,22 @@ in
     sha256 = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
   };
 
-  ibis310 = pkgs.callPackage ./ibis.nix {
-    python = pkgs.python310;
-    inherit mkEnv;
-  };
-
-  ibis311 = pkgs.callPackage ./ibis.nix {
-    python = pkgs.python311;
-    inherit mkEnv;
-  };
-
-  ibis312 = pkgs.callPackage ./ibis.nix {
-    python = pkgs.python312;
-    inherit mkEnv;
-  };
+  ibis310 = mkEnv pkgs.python310;
+  ibis311 = mkEnv pkgs.python311;
+  ibis312 = mkEnv pkgs.python312;
 
-  ibisDevEnv310 = mkEnv pkgs.python310 { };
-  ibisDevEnv311 = mkEnv pkgs.python311 { };
-  ibisDevEnv312 = mkEnv pkgs.python312 { };
+  ibisDevEnv310 = mkDevEnv pkgs.python310;
+  ibisDevEnv311 = mkDevEnv pkgs.python311;
+  ibisDevEnv312 = mkDevEnv pkgs.python312;
 
-  ibisSmallDevEnv = mkEnv pkgs.python312 {
-    deps = {
-      ibis-framework = [ "dev" ];
-    };
-  };
+  ibisSmallDevEnv = mkEnv'
+    {
+      deps = {
+        ibis-framework = [ "dev" ];
+      };
+      editable = false;
+    }
+    pkgs.python312;
 
   duckdb = super.duckdb.overrideAttrs (
     _: lib.optionalAttrs (stdenv.isAarch64 && stdenv.isLinux) {

nix/tests.nix

@@ -0,0 +1,41 @@
+{ pkgs, deps }:
+let
+  inherit (pkgs) stdenv;
+in
+final: prev: {
+  ibis-framework = prev.ibis-framework.overrideAttrs (old: {
+    passthru = old.passthru // {
+      tests = old.passthru.tests or { } // {
+        pytest =
+          let
+            pythonEnv = final.mkVirtualEnv "ibis-framework-test-env" (deps // {
+              # Use default dependencies from overlay.nix + enabled tests group.
+              ibis-framework = deps.ibis-framework or [ ] ++ [ "tests" ];
+            });
+          in
+          stdenv.mkDerivation {
+            name = "ibis-framework-test";
+            nativeCheckInputs = [ pythonEnv pkgs.graphviz-nox ];
+            src = ../.;
+            doCheck = true;
+            preCheck = ''
+              set -euo pipefail
+
+              ln -s ${pkgs.ibisTestingData} $PWD/ci/ibis-testing-data
+
+              HOME="$(mktemp -d)"
+              export HOME
+            '';
+            checkPhase = ''
+              runHook preCheck
+              pytest -m datafusion
+              pytest -m 'core or duckdb or sqlite or polars' --numprocesses $NIX_BUILD_CORES --dist loadgroup
+              runHook postCheck
+            '';
+
+            installPhase = "mkdir $out";
+          };
+      };
+    };
+  });
+}