feat: per-fit status record, failure record and run-state vocabulary (hpc-campaign phase 1)

autofit/config/general.yaml

@@ -7,6 +7,8 @@ hpc:
   hpc_mode: false                   # If True, use HPC mode, which disables GUI visualization, logging to screen and other settings which are not suited to running on a super computer.
   iterations_per_quick_update: 1e99 #  Non-linear search iterations between every quick update, which just displays the maximum likelihood model fit.
   iterations_per_full_update: 1e99  # Non-linear search iterations between every full update, which outputs all visuals and result fits (e.g. model.result, search.summary), this exits the search and can be slow.
+  status: true                      # If True, every fit writes a status record beside its output folder (<output_path>.status.json, at most 2 KB; .status.jsonl attempt history; .traceback.txt on failure) and flushes its checkpoint on SIGUSR1 / SIGTERM. If False, nothing is written and no signal handlers are installed.
+  status_interval_s: 60             # Seconds between the status record's cadence writes while a fit runs (it is also written at start, at every full update, on a signal, on completion and on failure).
 inversion:
   check_reconstruction: true        # If True, the inversion's reconstruction is checked to ensure the solution of a meshs's mapper is not an invalid solution where the values are all the same.
   reconstruction_vmax_factor: 0.5   # Plots of an Inversion's reconstruction use the reconstructed data's bright value multiplied by this factor.

autofit/non_linear/search/abstract_search.py

@@ -44,6 +44,7 @@
 from autofit.non_linear.search.execution import SearchExecutionMixin, _SearchMeta
 from autofit.non_linear.search.internal_state import SearchInternalStateMixin
 from autofit.non_linear.search.updates import SearchUpdatesMixin
+from autofit.non_linear.status import status_notify, status_recorded
 from autofit.non_linear.search.test_mode_bypass import TestModeBypass, TestModeMixin
 from autofit.non_linear.test_mode import test_mode_level, skip_fit_output
 
@@ -215,6 +216,7 @@ def __getstate__(self):
         state.pop("_logger", None)
         state.pop("paths", None)
         state.pop("_fit_pools", None)
+        state.pop("_fit_status", None)
         return state
 
     @property
@@ -286,6 +288,11 @@ class represented by model M and gives a score for their fitness: the search
         updated model whose priors represent the beliefs produced by this fit. Raises
         ``AssertionError`` if the model has 0 dimensions.
         """
+        with status_recorded(self):
+            return self._fit_and_output(model=model, analysis=analysis, info=info)
+
+    def _fit_and_output(self, model, analysis, info=None):
+        """The body of ``fit``, inside its status record (``status_recorded``)."""
         self.check_model(model=model)
 
         # Per-fit: set by `start_resume_fit` when the fit's pools are resolved, and read
@@ -394,6 +401,8 @@ def start_resume_fit(self, analysis: Analysis, model: AbstractPriorModel) -> Res
         if self.paths.has_timer:
             self.timer.start()
 
+        status_notify(self, "start")
+
         mode = test_mode_level()
         if mode >= 2:
             return TestModeBypass(self).run(
@@ -494,6 +503,8 @@ def _complete_fit(self, search_internal):
 
         self.paths.completed()
 
+        status_notify(self, "completed", search_internal)
+
     def result_via_completed_fit(
         self,
         analysis: Analysis,
@@ -511,6 +522,8 @@ def result_via_completed_fit(
         recreates the output files of the model-fit.
 
         """
+        status_notify(self, "skipped")
+
         model.freeze()
         samples_summary = self.paths.load_samples_summary()
         try:

autofit/non_linear/search/fit_context.py

@@ -28,6 +28,7 @@
 
 from autofit.non_linear.samples.adapter import RawSamples, samples_from_raw
 from autofit.non_linear.search.seed import seed_sequence
+from autofit.non_linear.status import status_notify
 
 
 class UpdateSchedule:
@@ -141,6 +142,10 @@ def update(self, internal: Any):
         if resume_state is not None:
             resume_state.save(self.paths, internal)
 
+        # The fit's status record takes the search's `progress` and keeps `internal`
+        # for a SIGUSR1 / SIGTERM checkpoint flush (`autofit.non_linear.status`).
+        status_notify(self._search, "update", internal)
+
         return self._search.perform_update(
             model=self.model,
             analysis=self._analysis,

autofit/non_linear/search/mcmc/emcee/search.py

@@ -139,6 +139,18 @@ def __init__(
 
         self.logger.debug("Creating Emcee Search")
 
+    def progress(self, search_internal) -> Optional[Dict]:
+        """
+        The run's progress for the status record (``autofit.non_linear.status``): the
+        steps taken of ``nsteps`` (a run that converges early stops before).
+        """
+        iteration = int(search_internal.iteration)
+        return {
+            "n_like": iteration * self.nwalkers,
+            "n_like_max": self.nsteps * self.nwalkers,
+            "fraction": min(1.0, iteration / self.nsteps) if self.nsteps else None,
+        }
+
     def run(self, ctx):
         """
         Run emcee (the ``run(ctx)`` hook; see ``docs/design/run_ctx.md``) and return the

autofit/non_linear/search/mle/bfgs/search.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Optional, TYPE_CHECKING
+from typing import Dict, Optional, TYPE_CHECKING
 
 from autofit import exc
 
@@ -224,6 +224,18 @@ def fitness_overrides(self, analysis: Analysis) -> dict:
         """
         return {"store_history": self.should_plot_start_point}
 
+    def progress(self, search_internal) -> Optional[Dict]:
+        """
+        The run's progress for the status record (``autofit.non_linear.status``): the
+        iterations taken of ``maxiter`` (scipy stops earlier once converged).
+        """
+        total = search_internal.get("total_iterations", search_internal.nit)
+        return {
+            "n_like": int(total),
+            "n_like_max": int(self.maxiter),
+            "fraction": min(1.0, total / self.maxiter) if self.maxiter else None,
+        }
+
     def run(self, ctx):
         """
         Run scipy's ``minimize`` (the ``run(ctx)`` hook; see ``docs/design/run_ctx.md``)

autofit/non_linear/search/nest/dynesty/search/abstract.py

@@ -451,6 +451,24 @@ def search_internal(self):
         """
         return self.resume_state.load(self.paths)
 
+    def progress(self, search_internal) -> Optional[Dict]:
+        """
+        The run's progress for the status record (``autofit.non_linear.status``): the
+        likelihood calls so far, the evidence and, when ``maxcall`` bounds the run, the
+        share of it spent (``fraction``).
+        """
+        progress = {
+            "n_like": self.total_calls_from(search_internal=search_internal),
+            "n_like_max": self.maxcall,
+        }
+        try:
+            progress["log_z"] = float(search_internal.results.logz[-1])
+        except Exception:
+            pass
+        if self.maxcall:
+            progress["fraction"] = min(1.0, progress["n_like"] / self.maxcall)
+        return progress
+
     def iterations_from(
         self, search_internal: "Union[NestedSampler, DynamicNestedSampler]"
     ) -> Tuple[int, int]:

autofit/non_linear/search/nest/nautilus/search.py

@@ -857,6 +857,28 @@ def call_search(self, search_internal, model, analysis, fitness, update=None):
 
         return search_internal
 
+    def progress(self, search_internal) -> Optional[Dict]:
+        """
+        The run's progress for the status record (``autofit.non_linear.status``): the
+        sampler's likelihood calls, live points, evidence, live-set fraction and
+        effective sample size; ``fraction`` is the share of ``n_like_max`` spent (none
+        when it is unbounded, since Nautilus stops on ``n_eff`` / ``f_live``).
+        """
+        progress = {
+            "n_like": int(search_internal.n_like),
+            "n_like_max": None,
+            "n_live": int(search_internal.n_live),
+        }
+        for key in ("log_z", "f_live", "n_eff"):
+            try:
+                progress[key] = float(getattr(search_internal, key))
+            except Exception:
+                pass
+        if self.n_like_max is not None and self.n_like_max != float("inf"):
+            progress["n_like_max"] = int(self.n_like_max)
+            progress["fraction"] = min(1.0, progress["n_like"] / self.n_like_max)
+        return progress
+
     def iterations_from(
         self, search_internal, iterations_per_full_update: Optional[float] = None
     ) -> Tuple[int, int]:

autofit/non_linear/status/__init__.py

@@ -0,0 +1,36 @@
+"""
+The per-fit status record (hpc-campaign phase 1).
+
+Every fit with an output folder writes ``<output_path>.status.json`` beside that folder
+(so it survives ``output.remove_files``): a compact (at most 2 KB), atomically replaced
+document with the fit's state, attempt, progress, resources and scheduler job, written
+at start, every ``general.hpc.status_interval_s`` seconds, at each full update, on
+SIGUSR1 / SIGTERM, on completion and on failure. ``<output_path>.status.jsonl`` keeps
+the attempt history and ``<output_path>.traceback.txt`` the last failure's traceback.
+``general.hpc.status: false`` turns all of it off.
+
+The ``progress`` protocol: a search may define ``progress(search_internal)``, returning
+a dict of whichever of ``n_like``, ``n_like_max``, ``n_live``, ``log_z``,
+``max_log_likelihood``, ``f_live``, ``n_eff`` and ``fraction`` (0-1) it knows, or
+``None``. It is called at each full update (``FitContext.update``); the record adds
+``eta_s`` from the fraction's rate.
+
+``states`` is the single vocabulary of run states, failure classes and product names.
+"""
+from autofit.non_linear.status.states import (
+    COLLECTOR_STATES,
+    TERMINAL_STATES,
+    WRITTEN_STATES,
+    FailureClass,
+    Product,
+    RunState,
+    failure_class_of,
+)
+from autofit.non_linear.status.writer import MAX_BYTES, SCHEMA, StatusWriter
+from autofit.non_linear.status.record import (
+    FitStatus,
+    status_enabled,
+    status_notify,
+    status_recorded,
+    status_interval_s,
+)