Run the FieldWorks parser conformance suite against HermitCrab in CI

.csharpierignore

@@ -2,4 +2,6 @@
 *.csproj
 *.props
 *.targets
-*.xml
+*.xml
+/conformance/
+/conformance.tmp-*/

.github/workflows/conformance.yml

@@ -0,0 +1,84 @@
+name: Conformance
+
+on:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "conformance.commit"
+      - "src/SIL.Machine/**"
+      - "src/SIL.Machine.Morphology.HermitCrab*/**"
+      - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**"
+      - ".github/workflows/conformance.yml"
+  pull_request:
+    paths:
+      - "conformance.commit"
+      - "src/SIL.Machine/**"
+      - "src/SIL.Machine.Morphology.HermitCrab*/**"
+      - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**"
+      - ".github/workflows/conformance.yml"
+
+jobs:
+  conformance:
+    name: Conformance suite
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: |
+            10.0.x
+
+      # Building the runner clones sillsdev/fieldworks-parser-conformance into conformance/ at the commit in
+      # conformance.commit; see EnsureConformanceSuite.targets.
+      #
+      # Release, because edge-cases/deep-optional-affix-nesting has a 15 s wall-clock budget that Debug uses
+      # most of on a small runner. Built once; every step below runs the built runner.
+      - name: Build the runner
+        run: dotnet build -c Release src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj
+
+      # Every fixture's words.yaml must match what the C# engine produces, in process, including the fixtures
+      # with a budget_ms.
+      - name: Run conformance self-check
+        run: >-
+          dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --
+          --fixtures conformance --include-pathological
+
+      # The suite's stress fixtures (conformance/stress) must agree with the engine too.
+      - name: Run stress self-check
+        if: always()
+        run: >-
+          dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --
+          --fixtures conformance/stress --include-pathological
+
+      # The suite's generated manifest must match a fresh computation, and every grammar and words.yaml must
+      # validate against the suite's DTD and schema.
+      - name: Check the suite manifest
+        if: always()
+        run: >-
+          dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --
+          --fixtures conformance --check-manifest
+
+      # Exits 1 on an uncovered construct, on a dead rule (exercised by zero words) missing from
+      # conformance/dead-rules-allowed.tsv, on an allowed rule some word now exercises, and when the suite's
+      # coverage.csv or rules.csv is stale. It writes nothing; the suite owns both files.
+      - name: Run coverage report
+        if: always()
+        run: >-
+          dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --
+          --fixtures conformance --coverage-report
+
+      # A pin that is not on the suite's main could be lost when that branch is deleted.
+      - name: Check the pinned commit is on the suite's main
+        if: always()
+        run: |
+          pin="$(tr -d '[:space:]' < conformance.commit)"
+          git -C conformance fetch --quiet origin main
+          if ! git -C conformance merge-base --is-ancestor "$pin" origin/main; then
+            echo "::error file=conformance.commit::$pin is not an ancestor of origin/main in the conformance suite"
+            exit 1
+          fi

.gitignore

@@ -58,3 +58,7 @@ tests/SIL.Machine.Tests/Corpora/TestData/pretranslations.json
 
 # Local, generated review summaries
 .review/
+
+# The conformance suite, cloned here at the commit in conformance.commit by the build
+/conformance/
+/conformance.tmp-*/

AGENTS.md

@@ -78,3 +78,11 @@ first row wins:
 
 `docs/review/devils-advocate.md` is an optional adversarial pass for a high-risk
 change. Add a nested `AGENTS.md` only when a subtree needs different rules.
+
+`conformance/` is an ignored clone of sillsdev/fieldworks-parser-conformance at
+the commit in `conformance.commit`; do not edit it from machine. Building the
+HermitCrab tests fetches it and moves it to the pinned commit after a pull or
+branch switch. Changes to the grammars land in that repository, following its
+`AGENTS.md`. To try an unmerged suite change, set `HC_CONFORMANCE_SUITE=<path>`.
+How to get, bump and check the suite from machine is in
+`src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md`.

Machine.sln

@@ -45,6 +45,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.Se
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.SentencePiece.Tests", "tests\SIL.Machine.Tokenization.SentencePiece.Tests\SIL.Machine.Tokenization.SentencePiece.Tests.csproj", "{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Morphology.HermitCrab.Conformance", "src\SIL.Machine.Morphology.HermitCrab.Conformance\SIL.Machine.Morphology.HermitCrab.Conformance.csproj", "{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -99,6 +101,10 @@ Global
 		{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -116,6 +122,7 @@ Global
 		{0BBEED18-3CC1-4BA6-8E11-603A5D2DEDC9} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3}
 		{09135223-6285-4268-BFA3-147BDB5B663B} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3}
 		{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC} = {66246A1C-8D45-40FB-A660-C58577122CA7}
+		{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}

README.md

@@ -22,6 +22,8 @@ Machine provides implementations of many common statistical word alignment model
 
 Machine contains a rule-based morphological/phonological parser called HermitCrab.
 
+The HermitCrab conformance grammars are in [sillsdev/fieldworks-parser-conformance](https://github.com/sillsdev/fieldworks-parser-conformance), an engine-neutral suite of grammars a FieldWorks project can produce, each with the parses an engine must return. Machine runs them from a clone in `conformance/`, at the commit named in `conformance.commit`, which the build fetches (`dotnet build` or `dotnet test` of the HermitCrab tests), with the runner in [`src/SIL.Machine.Morphology.HermitCrab.Conformance`](src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md).
+
 ### Feature Structures
 
 Machine provides a flexible implementation of feature structures with efficient unification, subsumption, and priority union operations. Feature values can be atomic symbols, strings, or variables.

conformance.commit

@@ -0,0 +1 @@
+7a0f7419fc706425171080ddbe3db263197b2d0d

local_check.sh

@@ -30,12 +30,40 @@ fi
 # output read through tail cannot pass off a build failure as a test pass.
 failed=()
 
+# conformance/ is the suite clone at the commit in conformance.commit; the build below clones it or moves it to the
+# pin. HC_CONFORMANCE_SUITE=<path> uses another copy.
+suite="${HC_CONFORMANCE_SUITE:-conformance}"
+
 dotnet tool restore || failed+=(tool-restore)
 dotnet restore || failed+=(restore)
 dotnet csharpier check . || failed+=(csharpier)
 dotnet build --no-restore -c Release || failed+=(build)
 dotnet test --verbosity normal || failed+=(test)
 
+# What the suite looks like before the runner steps, so edits you made to it do not read as the runner's.
+suite_state() {
+  git -C "$suite" status --porcelain --untracked-files=all 2>/dev/null
+  git -C "$suite" diff 2>/dev/null
+}
+suite_before=$(suite_state | sha1sum)
+
+# The same checks as .github/workflows/conformance.yml.
+hc_conformance=(dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --)
+"${hc_conformance[@]}" --fixtures "$suite" --include-pathological || failed+=(conformance)
+"${hc_conformance[@]}" --fixtures "$suite/stress" --include-pathological || failed+=(conformance-stress)
+"${hc_conformance[@]}" --fixtures "$suite" --check-manifest || failed+=(manifest)
+"${hc_conformance[@]}" --fixtures "$suite" --coverage-report || failed+=(coverage-report)
+# The runner steps must not have changed the suite, tracked or untracked.
+[ "$(suite_state | sha1sum)" = "$suite_before" ] || failed+=(suite-modified)
+# The pin must be on the suite's main (CI checks the same). Not checked without a network or with another suite.
+if [ -z "${HC_CONFORMANCE_SUITE:-}" ]; then
+  if git -C conformance fetch --quiet origin main; then
+    git -C conformance merge-base --is-ancestor "$(tr -d '[:space:]' < conformance.commit)" origin/main || failed+=(pin-not-on-main)
+  else
+    echo "local_check: could not fetch the suite's main, so the pin was not checked against it"
+  fi
+fi
+
 if [ "${#failed[@]}" -ne 0 ]; then
   echo "local_check: FAILED (${failed[*]})"
   exit 1

src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace SIL.Machine.Morphology.HermitCrab.Conformance;
+
+public class AdapterEngine : IEngine
+{
+    private readonly List<string> _templateTokens;
+    private readonly int _timeoutMs;
+
+    public AdapterEngine(string commandTemplate, IReadOnlySet<string> capabilities, int timeoutMs = 5 * 60 * 1000)
+    {
+        _templateTokens = new List<string>(SIL.Machine.Morphology.HermitCrab.Program.SplitCommandLine(commandTemplate));
+        if (_templateTokens.Count == 0)
+            throw new InvalidOperationException("empty --adapter command template");
+        _timeoutMs = timeoutMs;
+        Capabilities = capabilities;
+    }
+
+    public string Name => "adapter";
+
+    public IReadOnlySet<string> Capabilities { get; }
+
+    public List<TsvRow> Run(MaterializedFixture fixture)
+    {
+        string outputPath = Path.Combine(Path.GetTempPath(), "hc-conformance-" + Guid.NewGuid().ToString("N") + ".tsv");
+        try
+        {
+            // A fixture with a morpher: block reaches an adapter only through {morpher}; without it the
+            // adapter would run at the defaults the fixture overrides.
+            if (!fixture.Manifest.Morpher.IsEmpty && !_templateTokens.Exists(t => t.Contains("{morpher}")))
+            {
+                throw new InvalidOperationException(
+                    $"fixture '{fixture.Id}' sets morpher: settings but the --adapter command template has no "
+                        + "{morpher} placeholder to pass them through; add it or drop the morpher-settings capability"
+                );
+            }
+            var tokens = new List<string>(_templateTokens.Count);
+            foreach (string token in _templateTokens)
+            {
+                tokens.Add(
+                    token
+                        .Replace("{grammar}", Path.GetFullPath(fixture.GrammarPath))
+                        .Replace("{words}", Path.GetFullPath(fixture.WordsPath))
+                        .Replace("{output}", outputPath)
+                        .Replace("{morpher}", Path.GetFullPath(fixture.MorpherPath))
+                );
+            }
+
+            var psi = ChildProcessEnvironment.CreateStartInfo(tokens[0]);
+            psi.RedirectStandardOutput = true;
+            psi.RedirectStandardError = true;
+            for (int i = 1; i < tokens.Count; i++)
+                psi.ArgumentList.Add(tokens[i]);
+
+            using Process process =
+                Process.Start(psi)
+                ?? throw new InvalidOperationException($"failed to start adapter process '{tokens[0]}'");
+
+            Task<string> stdoutTask = process.StandardOutput.ReadToEndAsync();
+            Task<string> stderrTask = process.StandardError.ReadToEndAsync();
+
+            int effectiveTimeoutMs =
+                fixture.Manifest.Budget != null ? (int)fixture.Manifest.Budget.WallClockMs : _timeoutMs;
+
+            bool exited = process.WaitForExit(effectiveTimeoutMs);
+            if (!exited)
+            {
+                try
+                {
+                    process.Kill(entireProcessTree: true);
+                }
+                catch (InvalidOperationException) { }
+                throw new TimeoutException(
+                    $"adapter process for fixture '{fixture.Id}' did not exit within {effectiveTimeoutMs}ms"
+                );
+            }
+
+            string stdout = stdoutTask.GetAwaiter().GetResult();
+            string stderr = stderrTask.GetAwaiter().GetResult();
+
+            if (process.ExitCode != 0)
+            {
+                throw new EngineCrashException(
+                    $"adapter process exited {process.ExitCode} for fixture '{fixture.Id}'.\nstdout:\n{stdout}\nstderr:\n{stderr}"
+                );
+            }
+            if (!File.Exists(outputPath))
+            {
+                throw new InvalidOperationException(
+                    $"adapter process for fixture '{fixture.Id}' did not produce its output file.\nstdout:\n{stdout}\nstderr:\n{stderr}"
+                );
+            }
+            return SignatureTsv.ReadFile(outputPath);
+        }
+        finally
+        {
+            try
+            {
+                if (File.Exists(outputPath))
+                    File.Delete(outputPath);
+            }
+            catch (IOException) { }
+        }
+    }
+}