Test-library fixes: overridable kernel start timeout, and a batching-proof diagnostic assertion

tests/app/Spec/Tests/Rust/Diagnostics.hs

@@ -24,11 +24,11 @@ tests = describe "Diagnostics" $ do
     [__i|struct A { a: u8, b: u8 }
          const a: A = A { a: 10, };
         |] $ \diagnostics ->
-      assertDiagnosticRanges' (L.sortBy (compare `on` (^. LSP.message)) diagnostics) [
+      -- Only rustc's, via flycheck. rust-analyzer's own diagnostics for this file are not
+      -- dependable: it never links a detached file into the crate graph on some runs, and
+      -- then publishes none of its own at all.
+      assertDiagnosticRangesFromSource' "rustc" (L.sortBy (compare `on` (^. LSP.message)) diagnostics) [
         (Range (Position 1 13) (Position 1 14), Just (InR "E0063"), "missing field `b` in initializer of `A`\nmissing `b`")
-        -- (Range (Position 1 6) (Position 1 7), Just (InR "non_upper_case_globals"), "Constant `a` should have UPPER_SNAKE_CASE name, e.g. `A`")
-        -- , (Range (Position 1 13) (Position 1 14), Just (InR "E0063"), "missing field `b` in initializer of `A`\nmissing `b`")
-        -- , (Range (Position 1 13) (Position 1 14), Just (InR "E0063"), "missing structure fields:\n- b\n")
         ]
 
   testDiagnostics "rust-analyzer" "main.ipynb" LanguageKind_Rust [__i|println!("Hello world");

tests/src/TestLib/JupyterRunnerContext.hs

@@ -83,6 +83,10 @@ type HasJupyterRunnerContext context = (
   , HasBaseContext context
   )
 
+-- | Seconds papermill waits for a kernel to come up.
+defaultStartTimeout :: Int
+defaultStartTimeout = 120
+
 type JupyterRunnerMonad m = (
   MonadBaseControl IO m
   , MonadUnliftIO m
@@ -112,7 +116,7 @@ testKernelStdout'' :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m
   ) => Text -> Text -> (Maybe Text -> ExampleT context m ()) -> ExampleT context m ()
 testKernelStdout'' kernel code cb = do
-  runKernelCode kernel code $ \_notebookFile _outputNotebookFile outFile _errFile -> do
+  runKernelCode defaultStartTimeout kernel code $ \_notebookFile _outputNotebookFile outFile _errFile -> do
     doesFileExist outFile >>= \case
       True -> liftIO (T.readFile outFile) >>= cb . Just
       False -> cb Nothing
@@ -123,8 +127,13 @@ testKernelStdout'' kernel code cb = do
 testKernelSucceeds :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m
   ) => Text -> Text -> SpecFree context m ()
-testKernelSucceeds kernel code = it [i|#{kernel} -- #{summarizeCode code} (no errors)|] $
-  notebookShouldSatisfy kernel code $ \(JupyterNotebook {..}) ->
+testKernelSucceeds = testKernelSucceeds' defaultStartTimeout
+
+testKernelSucceeds' :: (
+  HasJupyterRunnerContext context, JupyterRunnerMonad m
+  ) => Int -> Text -> Text -> SpecFree context m ()
+testKernelSucceeds' startTimeout kernel code = it [i|#{kernel} -- #{summarizeCode code} (no errors)|] $
+  notebookShouldSatisfy startTimeout kernel code $ \(JupyterNotebook {..}) ->
     case [(errorOutputEname, errorOutputEvalue) | CodeCell {..} <- notebookCells, ErrorOutput {..} <- codeOutputs] of
       [] -> return ()
       ((ename, evalue) : _) -> expectationFailure [i|Kernel produced an error output: #{ename}: #{evalue}|]
@@ -154,7 +163,7 @@ displayTextsShouldBe kernel code desired = displayDatasShouldSatisfy kernel code
 displayDatasShouldSatisfy :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m
   ) => Text -> Text -> ([Map MimeType A.Value] -> ExampleT context m ()) -> ExampleT context m ()
-displayDatasShouldSatisfy kernel code cb = notebookShouldSatisfy kernel code $ \(JupyterNotebook {..}) -> do
+displayDatasShouldSatisfy kernel code cb = notebookShouldSatisfy defaultStartTimeout kernel code $ \(JupyterNotebook {..}) -> do
   let outputs = mconcat [codeOutputs | CodeCell {..} <- notebookCells]
   cb ([displayDataData | DisplayDataOutput {..} <- outputs])
 
@@ -185,7 +194,7 @@ executeTextsShouldBe kernel code desired = executeResultsShouldSatisfy kernel co
 executeResultsShouldSatisfy :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m
   ) => Text -> Text -> ([Map MimeType A.Value] -> ExampleT context m ()) -> ExampleT context m ()
-executeResultsShouldSatisfy kernel code cb = notebookShouldSatisfy kernel code $ \(JupyterNotebook {..}) -> do
+executeResultsShouldSatisfy kernel code cb = notebookShouldSatisfy defaultStartTimeout kernel code $ \(JupyterNotebook {..}) -> do
   let outputs = mconcat [codeOutputs | CodeCell {..} <- notebookCells]
   cb ([executeResultData | ExecuteResultOutput {..} <- outputs])
 
@@ -211,9 +220,9 @@ summarizeCode code = code
 
 notebookShouldSatisfy :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m
-  ) => Text -> Text -> (JupyterNotebook -> ExampleT context m ()) -> ExampleT context m ()
-notebookShouldSatisfy kernel code cb = do
-  runKernelCode kernel code $ \notebookFile outputNotebookFile _outFile _errFile -> do
+  ) => Int -> Text -> Text -> (JupyterNotebook -> ExampleT context m ()) -> ExampleT context m ()
+notebookShouldSatisfy startTimeout kernel code cb = do
+  runKernelCode startTimeout kernel code $ \notebookFile outputNotebookFile _outFile _errFile -> do
     liftIO (A.eitherDecodeFileStrict outputNotebookFile) >>= \case
       Left err -> expectationFailure [i|Failed to decode notebook '#{notebookFile}': #{err}|]
       Right nb -> cb nb
@@ -223,8 +232,8 @@ runKernelCode :: (
   , JupyterRunnerMonad m
   , MonadReader context m
   , MonadLoggerIO m
-  ) => Text -> Text -> (FilePath -> FilePath -> FilePath -> FilePath -> m b) -> m b
-runKernelCode kernel code cb = do
+  ) => Int -> Text -> Text -> (FilePath -> FilePath -> FilePath -> FilePath -> m b) -> m b
+runKernelCode startTimeout kernel code cb = do
   nixEnv <- getContext nixEnvironment
   let jupyterPath = nixEnv </> "lib" </> "codedown"
   debug [i|Got jupyterPath: #{jupyterPath}|]
@@ -261,7 +270,7 @@ runKernelCode kernel code cb = do
         "notebook.ipynb", "out.ipynb"
         , "--stdout-file", relativeToInnerRunDir outFile
         , "--stderr-file", relativeToInnerRunDir errFile
-        , "--start-timeout", "120"
+        , "--start-timeout", Prelude.show startTimeout
         , "--cwd", innerRunDir
         , "--log-level", "DEBUG"
         , "-k", T.unpack kernel

tests/src/TestLib/LSP.hs

@@ -16,6 +16,7 @@ module TestLib.LSP (
   , Helpers.getDiagnosticRanges'
   , assertDiagnosticRanges
   , assertDiagnosticRanges'
+  , assertDiagnosticRangesFromSource'
   , testDiagnostics
   , testDiagnosticsLabel
   , testDiagnosticsLabelDesired
@@ -216,3 +217,7 @@ assertDiagnosticRanges'' keyFn diagnostics desired = if
 
                               Found: #{A.encode $ keyFn diagnostics}
                              |]
+
+assertDiagnosticRangesFromSource' :: (HasCallStack, MonadIO m) => Text -> [Diagnostic] -> [(Range, Maybe (Int32 |? Text), Text)] -> m ()
+assertDiagnosticRangesFromSource' src diagnostics =
+  assertDiagnosticRanges' (L.filter (\d -> _source d == Just src) diagnostics)

tests/src/TestLib/VariableInspector.hs

@@ -170,7 +170,7 @@ runInspectorCommand :: (
   HasJupyterRunnerContext context, JupyterRunnerMonad m, FromJSON a
   ) => Text -> Text -> Text -> (a -> ExampleT context m ()) -> ExampleT context m ()
 runInspectorCommand kernel code what cb =
-  runKernelCode kernel code $ \notebookFile outputNotebook outFile _errFile -> do
+  runKernelCode defaultStartTimeout kernel code $ \notebookFile outputNotebook outFile _errFile -> do
     stdoutContents <- doesFileExist outFile >>= \case
       True -> liftIO (TIO.readFile outFile)
       False -> return ""