Compile the Rust kernel's crates at build time instead of at every startup

modules/kernels/rust/evcxr/default.nix

@@ -1,4 +1,6 @@
 { callPackage
+, coreutils
+, gnutar
 , lib
 , libiconv
 , makeWrapper
@@ -128,19 +130,14 @@ evcxr.overrideAttrs (oldAttrs: {
         # env-var overrides (LIBCLANG_PATH, ...). Values are store paths / flags
         # with no spaces, so they pass through $makeWrapperArgs word-splitting.
         envArgs = lib.concatLists (lib.mapAttrsToList (n: v: ["--set" n v]) env);
-      in
-      runCommand "evcxr" {
-        version = evcxr.version;
-
-        buildInputs = [makeWrapper];
 
         # evcxr links user code at *runtime* outside any Nix derivation, so the
         # native deps that stdenv would normally provide during a build are
         # missing. We bake into the wrapper:
         #   - the override-derived tools (pkg-config, cmake, ...) on PATH
         #   - pkg-config search paths for the override-derived libraries
         #   - libiconv on Darwin (rustc emits `-liconv` with no `-L` otherwise)
-        makeWrapperArgs = [
+        baseWrapperArgs = [
           "--set" "CARGO_HOME" "${cargoHome packages}"
           "--set" "EVCXR_CONFIG_DIR" "${evcxrConfigDir packages}"
           "--prefix" "PATH" ":" "${lib.makeBinPath ([rustc cargo] ++ nativeBuildInputs)}"
@@ -151,13 +148,82 @@ evcxr.overrideAttrs (oldAttrs: {
           "--set" "NIX_LDFLAGS_${stdenv.cc.suffixSalt}" "-L${lib.getLib libiconv}/lib"
         ];
 
+        # Compiles the dependencies itself, to produce the prebuilt directory.
+        unseeded = runCommand "evcxr-unseeded" {
+          buildInputs = [makeWrapper];
+          makeWrapperArgs = baseWrapperArgs;
+        } ''
+          mkdir -p $out/bin
+          makeWrapper ${evcxr}/bin/evcxr $out/bin/evcxr $makeWrapperArgs
+        '';
+
+        # Compile the crates once here rather than on every kernel start.
+        # EVCXR_TMPDIR makes the build directory outlive the process; a tarball
+        # keeps the mtimes cargo compares against, which the store would flatten.
+        prebuiltBuildDir = runCommand "evcxr-build-dir" {
+          nativeBuildInputs = [gnutar];
+        } ''
+          buildDir=$(mktemp -d)
+
+          # Cleared, so the rerun-if-env-changed variables match what a kernel
+          # sees; otherwise cargo reruns every build script.
+          echo '1+1' | env -i \
+            HOME=$(mktemp -d) \
+            TMPDIR=$TMPDIR \
+            EVCXR_TMPDIR=$buildDir \
+            ${unseeded}/bin/evcxr > evcxr.log 2>&1
+
+          # evcxr exits 0 even when the build fails; the expression can only have
+          # evaluated if the crates compiled.
+          if ! grep -qx '2' evcxr.log; then
+            echo "evcxr failed while prebuilding the dependencies:"
+            grep -v '"reason":"compiler-artifact"' evcxr.log | tail -50
+            exit 1
+          fi
+
+          mkdir -p $out
+          tar -cf $out/build-dir.tar -C "$buildDir" .
+        '';
+      in
+      runCommand "evcxr" {
+        version = evcxr.version;
+
+        buildInputs = [makeWrapper];
+
+        makeWrapperArgs = baseWrapperArgs;
+
         passthru = {
           cargoHome = cargoHome packages;
+          inherit prebuiltBuildDir;
         };
       } ''
         mkdir -p $out/bin
-        makeWrapper ${evcxr}/bin/evcxr $out/bin/evcxr $makeWrapperArgs
-        makeWrapper ${evcxr}/bin/evcxr_jupyter $out/bin/evcxr_jupyter $makeWrapperArgs
+
+        # Separate from $makeWrapperArgs, which is word-split. Absolute paths
+        # because the wrapper's PATH has the Rust toolchain, not coreutils.
+        seed=${lib.escapeShellArg ''
+          # Skipped when already set: evcxr re-execs itself through this wrapper,
+          # and a second build directory would deadlock the two on cargo's lock.
+          if [ -z "''${EVCXR_TMPDIR:-}" ]; then
+            EVCXR_TMPDIR=$(${coreutils}/bin/mktemp -d "''${TMPDIR:-/tmp}/evcxr-build-XXXXXX")
+            export EVCXR_TMPDIR
+
+            # Unpacking restores the mtimes cargo compares against.
+            ${gnutar}/bin/tar -xf ${prebuiltBuildDir}/build-dir.tar -C "$EVCXR_TMPDIR"
+            ${coreutils}/bin/chmod -R u+w "$EVCXR_TMPDIR"
+
+            # This config wins over CARGO_HOME's, and evcxr leaves it saying
+            # offline = false, sending cargo after an index we already vendor.
+            printf '[net]\noffline = true\n' > "$EVCXR_TMPDIR/.cargo/config.toml"
+
+            # Both are empty, so tar stores them as one hardlinked inode and cargo
+            # deadlocks locking it twice. It recreates them.
+            ${coreutils}/bin/rm -f "$EVCXR_TMPDIR"/target/*/.cargo-lock "$EVCXR_TMPDIR"/target/*/*/.cargo-lock
+          fi
+        ''}
+
+        makeWrapper ${evcxr}/bin/evcxr $out/bin/evcxr $makeWrapperArgs --run "$seed"
+        makeWrapper ${evcxr}/bin/evcxr_jupyter $out/bin/evcxr_jupyter $makeWrapperArgs --run "$seed"
       '';
   };
 })

tests/app/Spec/Tests/Rust.hs

@@ -2,11 +2,15 @@
 
 module Spec.Tests.Rust (tests) where
 
+import Control.Monad (unless)
+import Control.Monad.IO.Class (liftIO)
 import Data.Aeson as A
 import Data.String.Interpolate
+import qualified Data.List as L
 import qualified Data.Text as T
 import qualified Data.Vector as V
 import Safe
+import System.FilePath ((</>))
 import Test.Sandwich as Sandwich
 import TestLib.JupyterRunnerContext
 import TestLib.NixEnvironmentContext
@@ -64,10 +68,27 @@ tests = describe "Rust" $ do
   -- dependencies that evcxr must compile at runtime. plotters in particular
   -- needs pkg-config + fontconfig/freetype, which the evcxr wrapper bakes in.
   -- This guards against regressions where the import block fails to compile.
+  --
+  -- It also covers the prebuilt crate artifacts: without them evcxr compiles
+  -- this set on startup, which takes longer than the kernel start timeout, so
+  -- this test fails outright rather than merely getting slower.
   introduceNixEnvironment [dataScienceKernelSpec] [] "Rust data science" $
-    introduceJupyterRunner $ describe "Data science kernel" $
+    introduceJupyterRunner $ describe "Data science kernel" $ do
       testKernelSucceeds "rust" dataScienceImports
 
+      -- The test above only notices a missing prebuild through the timeout, so
+      -- check the wiring directly too.
+      it "seeds a prebuilt build directory" $ do
+        nixEnv <- getContext nixEnvironment
+        Just (kernel :: A.Object) <- liftIO $ A.decodeFileStrict
+          (nixEnv </> "lib" </> "codedown" </> "kernels" </> "rust" </> "kernel.json")
+        case aesonLookup "argv" kernel of
+          Just (A.Array (V.toList -> (A.String evcxr : _))) -> do
+            wrapper <- liftIO $ readFile (T.unpack evcxr)
+            unless ("evcxr-build-dir" `L.isInfixOf` wrapper) $
+              expectationFailure [i|#{evcxr} doesn't seed a prebuilt build directory, so the kernel compiles its crates on every start|]
+          x -> expectationFailure [i|Expected an argv in the kernel spec, got: #{x}|]
+
 -- rand 0.9+ dropped the `random` re-export from `rand::prelude`, so call it by full path
 -- instead of `use rand::prelude::*; let x: u8 = random();`. See issue #49.
 randCode :: T.Text