From 3c364a1c786b5e6790aa92c2cf1f0392660d2306 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 12:43:18 -0500 Subject: [PATCH 01/15] Add a batch command to the HermitCrab tool The tool's batch command parses every word of a list with one grammar and writes one row per word: index, word, milliseconds, status and result signature. SignatureFormat builds the signature, so an external runner compares its engine against the same text. The script mode of the tool now returns the first nonzero command result instead of always 0, and SplitCommandLine is internal so a runner tokenizes a command template the same way. A MorpherTests case pins the Morpher constructor defaults that the signature protocol documents. Co-Authored-By: Claude Sonnet 5.5 --- .../BatchCommand.cs | 85 +++++++++++++++++++ .../Program.cs | 16 +++- .../Properties/AssemblyInfo.cs | 4 + .../SignatureFormat.cs | 57 +++++++++++++ .../MorpherTests.cs | 11 +++ 5 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs new file mode 100644 index 000000000..2eaa313c7 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs @@ -0,0 +1,85 @@ +using System.Diagnostics; +using System.Globalization; +using System.IO; +using ManyConsole; +using Mono.Options; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Parses every word in a word list and records a per-word result signature plus elapsed time, so two +/// runs (for example, before and after an engine change) can be diffed to confirm parse results are +/// unchanged. Output is flushed per word, and --start resumes a run after a crash, because a single word +/// can take minutes. +/// +internal class BatchCommand : ConsoleCommand +{ + private readonly HCContext _context; + private int _startIndex; + + public BatchCommand(HCContext context) + { + _context = context; + + IsCommand("batch", "Parses every word in a word list, recording a result signature and timing per word"); + SkipsCommandSummaryBeforeRunning(); + HasAdditionalArguments(2, " "); + HasOption("start=", "0-based line index to resume at (for crash recovery)", v => _startIndex = ParseStart(v)); + } + + /// NumberStyles.None rejects a sign, so a negative index fails here rather than at words[i]. + private static int ParseStart(string value) + { + if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out int start)) + throw new OptionException($"--start must be a non-negative integer, got '{value}'", "--start"); + return start; + } + + public override int Run(string[] remainingArguments) + { + string wordListPath = remainingArguments[0]; + string outputPath = remainingArguments[1]; + + try + { + string[] words = SignatureFormat.LoadWords(wordListPath); + return RunSequential(words, outputPath); + } + finally + { + // One instance serves every line of a script, and --start is parsed only when given. + _startIndex = 0; + } + } + + private int RunSequential(string[] words, string outputPath) + { + using var writer = new StreamWriter(outputPath, append: _startIndex > 0) { AutoFlush = true }; + var totalSw = Stopwatch.StartNew(); + long parsed = 0, + skipped = 0; + for (int i = _startIndex; i < words.Length; i++) + { + string word = words[i]; + // Sentinel written before the attempt: if this word crashes the process, a wrapper script + // can read the last line to find where to resume. + writer.WriteLine($"{i}\t{word}\tSTARTED"); + (string status, long elapsedMs, string signature) = SignatureFormat.ParseOneWord(_context.Morpher, word); + writer.WriteLine($"{i}\t{word}\t{elapsedMs}\t{status}\t{signature}"); + if (status == "SKIPPED") + skipped++; + else + parsed++; + if (i % 100 == 0) + _context.Out.WriteLine("[{0}/{1}]", i, words.Length); + } + totalSw.Stop(); + _context.Out.WriteLine( + "batch complete: {0} words parsed ({1} skipped), {2}ms total", + parsed, + skipped, + totalSw.ElapsedMilliseconds + ); + return 0; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs index ff8e86bcc..251e2f555 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs @@ -92,9 +92,11 @@ public static int Main(string[] args) new TracingCommand(context), new TestCommand(context), new StatsCommand(context), + new BatchCommand(context), }; string input; + int exitCode = 0; if (!string.IsNullOrEmpty(scriptFile)) { using (var scriptReader = new StreamReader(scriptFile)) @@ -105,7 +107,12 @@ public static int Main(string[] args) if (!input.Trim().StartsWith("#") && input.Trim() != "") { string[] cmdArgs = SplitCommandLine(input); - ConsoleCommandDispatcher.DispatchCommand(commands, cmdArgs, context.Out); + int result = ConsoleCommandDispatcher.DispatchCommand(commands, cmdArgs, context.Out); + // Remember the first nonzero result rather than the last: once a script + // line fails, later lines running (or not) shouldn't paper over that with + // a subsequent success. + if (result != 0 && exitCode == 0) + exitCode = result; } input = scriptReader.ReadLine(); } @@ -133,10 +140,13 @@ public static int Main(string[] args) output?.Close(); - return 0; + return exitCode; } - private static string[] SplitCommandLine(string commandLine) + /// Splits a command line into tokens, honoring double- and single-quoted segments. + /// Shared with SIL.Machine.Morphology.HermitCrab.Conformance's AdapterEngine via + /// InternalsVisibleTo so both consumers tokenize the exact same way. + internal static string[] SplitCommandLine(string commandLine) { var parmChars = commandLine.ToCharArray(); var inSingleQuote = false; diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..46d786444 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("hc-conformance")] +[assembly: InternalsVisibleTo("SIL.Machine.Morphology.HermitCrab.Tests")] diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs new file mode 100644 index 000000000..27ee72505 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab; + +internal static class SignatureFormat +{ + /// Reads a word list file: one word per line, trimmed, blank lines dropped. + public static string[] LoadWords(string path) + { + return File.ReadAllLines(path).Select(w => w.Trim()).Where(w => w.Length > 0).ToArray(); + } + + public static (string status, long elapsedMs, string signature) ParseOneWord(Morpher morpher, string word) + { + return ParseOneWord(morpher, word, guessRoot: false); + } + + public static (string status, long elapsedMs, string signature) ParseOneWord( + Morpher morpher, + string word, + bool guessRoot + ) + { + var sw = Stopwatch.StartNew(); + try + { + Word[] results = morpher.ParseWord(word, out _, guessRoot).ToArray(); + sw.Stop(); + return ("ok", sw.ElapsedMilliseconds, BuildSignature(results)); + } + catch (InvalidShapeException) + { + return ("SKIPPED", 0, "-"); + } + } + + /// + /// Builds the order-independent, sorted, semicolon-joined signature for a set of parse + /// results: one morph+morph|shape entry per distinct analysis, or "-" if empty. + /// + public static string BuildSignature(IEnumerable results) + { + List signatures = results + .Select(w => + string.Join("+", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Id)) + + "|" + + w.Shape.ToRegexString(w.Stratum.CharacterDefinitionTable, true) + ) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + return signatures.Count == 0 ? "-" : string.Join(";", signatures); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs index d9149d058..4028a425d 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs @@ -11,6 +11,17 @@ namespace SIL.Machine.Morphology.HermitCrab; [TestFixture] public class MorpherTests : HermitCrabTestBase { + [Test] + public void Constructor_DefaultsMatchDocumentedProtocolDefaults() + { + var morpher = new Morpher(TraceManager, Language); + + Assert.That(morpher.MaxStemCount, Is.EqualTo(2)); + Assert.That(morpher.DeletionReapplications, Is.EqualTo(0)); + Assert.That(morpher.MaxAlternatives, Is.EqualTo(0)); + Assert.That(morpher.MergeEquivalentAnalyses, Is.True); + } + [Test] public void AnalyzeWord_CanAnalyze_ReturnsCorrectAnalysis() { From 62c92550102bbc5aeb4a299474379f53dd2cc9dc Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 13:21:30 -0500 Subject: [PATCH 02/15] Pin the HermitCrab conformance suite to one commit conformance.commit holds the full SHA of the commit of sillsdev/fieldworks-parser-conformance that machine runs. The clone in conformance/ is ignored by git and by csharpier, as is the temporary directory a clone is made in. Co-Authored-By: Claude Sonnet 5.5 --- .csharpierignore | 4 +++- .gitignore | 4 ++++ conformance.commit | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 conformance.commit diff --git a/.csharpierignore b/.csharpierignore index b5df929f0..e8b74c9a6 100644 --- a/.csharpierignore +++ b/.csharpierignore @@ -2,4 +2,6 @@ *.csproj *.props *.targets -*.xml \ No newline at end of file +*.xml +/conformance/ +/conformance.tmp-*/ diff --git a/.gitignore b/.gitignore index 0437b6304..b7a33c859 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ tests/SIL.Machine.Tests/Corpora/TestData/pretranslations.json # Local, generated review summaries .review/ + +# The conformance suite, cloned here at the commit in conformance.commit by the build +/conformance/ +/conformance.tmp-*/ diff --git a/conformance.commit b/conformance.commit new file mode 100644 index 000000000..f4acce70d --- /dev/null +++ b/conformance.commit @@ -0,0 +1 @@ +7a0f7419fc706425171080ddbe3db263197b2d0d From 9c4be00ba384656de83d586125ad5be7931374ca Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 14:19:03 -0500 Subject: [PATCH 03/15] Add the hc-conformance runner and its tests The runner checks the conformance suite against the C# HermitCrab engine. Each fixture's words.yaml is compared with what the engine returns, word by word, in process, including traced rules and budget_ms. --adapter runs the same fixtures through an external engine from a command template, as the suite's PROTOCOL.md describes. --propose prints the words.yaml patch for a mismatch, --coverage-report checks the suite's coverage.csv, rules.csv and dead-rule list, and --check-manifest validates every grammar and words.yaml against the suite's DTD and schema and compares the suite's generated manifest with a fresh computation. The runner writes nothing into the suite. Building the runner runs EnsureConformanceSuite.targets, which makes conformance/ a clone of the suite at the commit in conformance.commit: cloned into a temporary directory and renamed when absent, moved to the pin when it is at another commit, left alone when it has uncommitted changes, and never fetched when it is already at the pin. A clone at the pin with local changes gets a warning naming them. A git that hangs is killed after three minutes, and any failure of the target is a warning, never a build failure. The runner warns when conformance/ is not at the pin or has local changes. HC_CONFORMANCE_SUITE= uses another copy instead. SIL.Machine, the HermitCrab library and the tool do not reference the runner, so they never run it. The tests run every fixture, the manifest, the schemas and the FieldWorks-producible shape rules in dotnet test, and fail with the command that fixes it when the suite is missing or not at the pin. A test pins the script-mode exit code of the hc tool, which is the first nonzero result. Co-Authored-By: Claude Sonnet 5.5 --- Machine.sln | 7 + .../AdapterEngine.cs | 109 +++ .../CanonicalJson.cs | 100 ++ .../ChildProcessEnvironment.cs | 36 + .../ConformanceManifestArtifact.cs | 45 + .../ConformanceManifestGenerator.cs | 141 +++ .../ConformanceMorpherFactory.cs | 22 + .../CoverageReport.cs | 243 +++++ .../Diff.cs | 104 ++ .../EngineCrashException.cs | 21 + .../EnsureConformanceSuite.targets | 254 +++++ .../Fixture.cs | 46 + .../FixtureManifest.cs | 35 + .../FixtureMaterializer.cs | 126 +++ .../GrammarRuleIndex.cs | 150 +++ .../GrammarValidation.cs | 75 ++ .../IEngine.cs | 19 + .../MaterializedFixture.cs | 38 + .../MaterializedRunner.cs | 286 ++++++ .../MorpherSettings.cs | 85 ++ .../Program.cs | 445 +++++++++ .../Properties/AssemblyInfo.cs | 3 + .../ProposePatchWriter.cs | 46 + .../RequiresDerivation.cs | 47 + .../Runner.cs | 434 +++++++++ ...e.Morphology.HermitCrab.Conformance.csproj | 32 + .../SelfCheckEngine.cs | 69 ++ .../SignatureTsv.cs | 75 ++ .../SuitePinCheck.cs | 98 ++ .../TraceRuleAttributor.cs | 100 ++ .../WordsSchemaValidation.cs | 142 +++ .../WordsYaml.cs | 94 ++ .../WordsYamlLoader.cs | 512 ++++++++++ .../BatchCommandTests.cs | 77 ++ .../ConformanceCommandLineTests.cs | 117 +++ .../ConformanceFixtureGateTests.cs | 353 +++++++ .../ConformanceHarnessTests.cs | 211 +++++ .../ConformanceManifestTests.cs | 302 ++++++ .../ConformanceMorpherFactoryTests.cs | 38 + .../ConformanceSuite.cs | 93 ++ .../DeadRuleAllowlistTests.cs | 156 +++ .../FieldWorksProducibleShapeTests.cs | 895 ++++++++++++++++++ .../FieldWorksWritableShapes.cs | 432 +++++++++ .../GuesserSignatureTests.cs | 112 +++ ...erializedRunnerCrashClassificationTests.cs | 136 +++ .../ProcessStartInfoConstructionGateTests.cs | 94 ++ .../ProposePatchWriterTests.cs | 37 + ...Machine.Morphology.HermitCrab.Tests.csproj | 1 + .../SuitePinCheckTests.cs | 104 ++ .../ToolScriptExitCodeTests.cs | 50 + .../WordsYamlMorpherSettingsTests.cs | 196 ++++ 51 files changed, 7443 insertions(+) create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs diff --git a/Machine.sln b/Machine.sln index 0d91d2dac..eb4ead14e 100644 --- a/Machine.sln +++ b/Machine.sln @@ -45,6 +45,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.Se EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.SentencePiece.Tests", "tests\SIL.Machine.Tokenization.SentencePiece.Tests\SIL.Machine.Tokenization.SentencePiece.Tests.csproj", "{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Morphology.HermitCrab.Conformance", "src\SIL.Machine.Morphology.HermitCrab.Conformance\SIL.Machine.Morphology.HermitCrab.Conformance.csproj", "{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -99,6 +101,10 @@ Global {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Debug|Any CPU.Build.0 = Debug|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.Build.0 = Release|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -116,6 +122,7 @@ Global {0BBEED18-3CC1-4BA6-8E11-603A5D2DEDC9} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {09135223-6285-4268-BFA3-147BDB5B663B} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC} = {66246A1C-8D45-40FB-A660-C58577122CA7} + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs new file mode 100644 index 000000000..c6c1ebfba --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public class AdapterEngine : IEngine +{ + private readonly List _templateTokens; + private readonly int _timeoutMs; + + public AdapterEngine(string commandTemplate, IReadOnlySet capabilities, int timeoutMs = 5 * 60 * 1000) + { + _templateTokens = new List(SIL.Machine.Morphology.HermitCrab.Program.SplitCommandLine(commandTemplate)); + if (_templateTokens.Count == 0) + throw new InvalidOperationException("empty --adapter command template"); + _timeoutMs = timeoutMs; + Capabilities = capabilities; + } + + public string Name => "adapter"; + + public IReadOnlySet Capabilities { get; } + + public List Run(MaterializedFixture fixture) + { + string outputPath = Path.Combine(Path.GetTempPath(), "hc-conformance-" + Guid.NewGuid().ToString("N") + ".tsv"); + try + { + // A fixture with a morpher: block reaches an adapter only through {morpher}; without it the + // adapter would run at the defaults the fixture overrides. + if (!fixture.Manifest.Morpher.IsEmpty && !_templateTokens.Exists(t => t.Contains("{morpher}"))) + { + throw new InvalidOperationException( + $"fixture '{fixture.Id}' sets morpher: settings but the --adapter command template has no " + + "{morpher} placeholder to pass them through; add it or drop the morpher-settings capability" + ); + } + var tokens = new List(_templateTokens.Count); + foreach (string token in _templateTokens) + { + tokens.Add( + token + .Replace("{grammar}", Path.GetFullPath(fixture.GrammarPath)) + .Replace("{words}", Path.GetFullPath(fixture.WordsPath)) + .Replace("{output}", outputPath) + .Replace("{morpher}", Path.GetFullPath(fixture.MorpherPath)) + ); + } + + var psi = ChildProcessEnvironment.CreateStartInfo(tokens[0]); + psi.RedirectStandardOutput = true; + psi.RedirectStandardError = true; + for (int i = 1; i < tokens.Count; i++) + psi.ArgumentList.Add(tokens[i]); + + using Process process = + Process.Start(psi) + ?? throw new InvalidOperationException($"failed to start adapter process '{tokens[0]}'"); + + Task stdoutTask = process.StandardOutput.ReadToEndAsync(); + Task stderrTask = process.StandardError.ReadToEndAsync(); + + int effectiveTimeoutMs = + fixture.Manifest.Budget != null ? (int)fixture.Manifest.Budget.WallClockMs : _timeoutMs; + + bool exited = process.WaitForExit(effectiveTimeoutMs); + if (!exited) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) { } + throw new TimeoutException( + $"adapter process for fixture '{fixture.Id}' did not exit within {effectiveTimeoutMs}ms" + ); + } + + string stdout = stdoutTask.GetAwaiter().GetResult(); + string stderr = stderrTask.GetAwaiter().GetResult(); + + if (process.ExitCode != 0) + { + throw new EngineCrashException( + $"adapter process exited {process.ExitCode} for fixture '{fixture.Id}'.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + } + if (!File.Exists(outputPath)) + { + throw new InvalidOperationException( + $"adapter process for fixture '{fixture.Id}' did not produce its output file.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + } + return SignatureTsv.ReadFile(outputPath); + } + finally + { + try + { + if (File.Exists(outputPath)) + File.Delete(outputPath); + } + catch (IOException) { } + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs new file mode 100644 index 000000000..6cc548b26 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs @@ -0,0 +1,100 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Produces compact UTF-8 JSON with recursively ordinal-sorted object keys. +internal static class CanonicalJson +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = null, + }; + + internal static byte[] SerializeUtf8(T value) + { + ArgumentNullException.ThrowIfNull(value); + using JsonDocument document = JsonDocument.Parse(JsonSerializer.SerializeToUtf8Bytes(value, SerializerOptions)); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false })) + WriteElement(document.RootElement, writer); + return stream.ToArray(); + } + + internal static string Serialize(T value) => Encoding.UTF8.GetString(SerializeUtf8(value)); + + private static void WriteElement(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + var properties = new List(); + var propertyNames = new HashSet(StringComparer.Ordinal); + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!propertyNames.Add(property.Name)) + throw new InvalidDataException($"Duplicate JSON property '{property.Name}'."); + properties.Add(property); + } + properties.Sort((left, right) => StringComparer.Ordinal.Compare(left.Name, right.Name)); + foreach (JsonProperty property in properties) + { + writer.WritePropertyName(property.Name); + WriteElement(property.Value, writer); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in element.EnumerateArray()) + WriteElement(item, writer); + writer.WriteEndArray(); + break; + case JsonValueKind.String: + writer.WriteStringValue(NormalizeString(element.GetString()!)); + break; + case JsonValueKind.Number: + if ( + !decimal.TryParse( + element.GetRawText(), + NumberStyles.Float, + CultureInfo.InvariantCulture, + out decimal decimalValue + ) + ) + { + throw new InvalidDataException( + $"JSON number '{element.GetRawText()}' is outside the supported canonical range." + ); + } + writer.WriteRawValue( + decimalValue.ToString("G29", CultureInfo.InvariantCulture), + skipInputValidation: true + ); + break; + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + default: + throw new InvalidDataException($"Unsupported JSON value kind '{element.ValueKind}'."); + } + } + + private static string NormalizeString(string value) => + value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs new file mode 100644 index 000000000..b7ff31da2 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs @@ -0,0 +1,36 @@ +#nullable enable +using System.Diagnostics; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// The only place this assembly (and the test assembly it grants internals to) may construct a child +/// ; ProcessStartInfoConstructionGateTests fails naming any other +/// file that does. Xplat Code Coverage attaches a CLR profiler to the current process via +/// CORECLR_*/COR_* environment variables, which ProcessStartInfo inherits into a child by default, and +/// a profiled child that then crashes or is killed has been observed to destabilize the coverage +/// channel the parent test host shares -- reproduced twice as a trace-less test host crash under +/// `--collect:"Xplat Code Coverage"` before this factory existed. +/// +internal static class ChildProcessEnvironment +{ + private static readonly string[] ProfilerVariableNames = + { + "CORECLR_ENABLE_PROFILING", + "CORECLR_PROFILER", + "CORECLR_PROFILER_PATH", + "CORECLR_PROFILER_PATH_32", + "CORECLR_PROFILER_PATH_64", + "COR_ENABLE_PROFILING", + "COR_PROFILER", + "COR_PROFILER_PATH", + }; + + internal static ProcessStartInfo CreateStartInfo(string fileName) + { + var startInfo = new ProcessStartInfo { FileName = fileName, UseShellExecute = false }; + foreach (string name in ProfilerVariableNames) + startInfo.EnvironmentVariables.Remove(name); + return startInfo; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs new file mode 100644 index 000000000..39899895b --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs @@ -0,0 +1,45 @@ +#nullable enable +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Provenance and integrity for the suite's conformance fixtures. +/// Deliberately not a second copy of the ground truth. words.yaml is the single +/// canonical authored format -- it carries the reasoning for each fixture in comments, which no +/// generated JSON can hold -- and grammar.xml is the authoritative grammar. This manifest +/// records what those files are, which format each conforms to, and the hash a consumer verifies +/// them against. +public sealed record ConformanceManifest( + string FormatVersion, + string WordsAuthoringFormatVersion, + string GrammarFormatVersion, + string DtdPath, + string DtdSha256, + string SourceHash, + IReadOnlyList Fixtures +) +{ + public const string ManifestFormat = "hc-conformance-manifest/v2"; + public const string WordsFormat = "hc-conformance-words/v1"; + public const string GrammarFormat = "sil-machine-hermit-crab-input-xml/v1"; +} + +/// Cases authored in words.yaml, so a consumer can detect a partial +/// read without parsing the file. +public sealed record ManifestFixture( + string FixtureId, + string Category, + string DisplayLanguage, + string GrammarPath, + string GrammarSha256, + string WordsPath, + string WordsSha256, + int CaseCount, + bool ExpectedCrash +); + +/// Canonical serialization of the manifest. +public static class ManifestJson +{ + public static string Serialize(ConformanceManifest manifest) => CanonicalJson.Serialize(manifest); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs new file mode 100644 index 000000000..b7d2627b0 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs @@ -0,0 +1,141 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Builds the from the fixtures under a suite root. +/// Nothing here interprets a grammar or restates a words file; each contributes a +/// suite-relative path and a hash. The authored files stay the only ground truth. +public static class ConformanceManifestGenerator +{ + /// The suite's manifest, relative to the suite root. The suite's generator writes it. + public const string ManifestRelativePath = "generated/hc-conformance-manifest.v2.json"; + + /// The DTD as the manifest names it, relative to the suite root. + public const string DtdRelativePath = "HermitCrabInput.dtd"; + + /// Files the manifest's own validity depends on, beyond the fixtures themselves, relative to + /// the suite root. + private static readonly string[] AdditionalSourceFiles = + { + "constructs.txt", + "parity-check.py", + "schema/words.schema.json", + "schema/conformance-manifest.schema.json", + }; + + public static ConformanceManifest Generate(string suiteRoot) + { + ArgumentException.ThrowIfNullOrWhiteSpace(suiteRoot); + string root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(suiteRoot)); + string dtdRelative = DtdRelativePath; + string dtdHash = HashFile(Path.Combine(root, dtdRelative.Replace('/', Path.DirectorySeparatorChar))); + + List fixtures = Fixture.DiscoverAll(root); + var mapped = fixtures + .Select(fixture => MapFixture(fixture, root)) + .OrderBy(fixture => fixture.FixtureId, StringComparer.Ordinal) + .ToArray(); + + return new ConformanceManifest( + ConformanceManifest.ManifestFormat, + ConformanceManifest.WordsFormat, + ConformanceManifest.GrammarFormat, + dtdRelative, + dtdHash, + SourceHash(root, mapped, dtdRelative, dtdHash), + mapped + ); + } + + private static ManifestFixture MapFixture(Fixture fixture, string root) + { + string fixtureId = RelativePath(fixture.Directory, root); + string grammarPath = fixtureId + "/grammar.xml"; + string wordsPath = fixtureId + "/words.yaml"; + WordsYaml words = fixture.Words; + + string[] duplicates = words + .Words.Select(word => word.Word) + .GroupBy(input => input, StringComparer.Ordinal) + .Where(group => group.Count() > 1) + .Select(group => group.Key) + .OrderBy(input => input, StringComparer.Ordinal) + .ToArray(); + if (duplicates.Length != 0) + { + throw new InvalidDataException( + $"Fixture '{fixtureId}' repeats input(s): {string.Join(", ", duplicates)}. Inputs must be unique within a fixture." + ); + } + + if (words.ExpectCrash && words.Words.Count != 1) + { + throw new InvalidDataException( + $"Fixture '{fixtureId}' declares expect_crash and must contain exactly one case, but has {words.Words.Count}." + ); + } + + return new ManifestFixture( + fixtureId, + fixtureId.StartsWith("languages/", StringComparison.Ordinal) ? "language" : "edge-case", + words.Language, + grammarPath, + HashFile(Path.Combine(root, grammarPath.Replace('/', Path.DirectorySeparatorChar))), + wordsPath, + HashFile(Path.Combine(root, wordsPath.Replace('/', Path.DirectorySeparatorChar))), + words.Words.Count, + words.ExpectCrash + ); + } + + private static string RelativePath(string path, string basePath) => + Path.GetRelativePath(basePath, path).Replace('\\', '/'); + + /// Hash of every file the manifest's validity rests on. + /// Path- and EOL-invariant: only suite-relative paths and LF-normalized text + /// participate, so relocating the checkout or changing line endings cannot move it. + private static string SourceHash( + string root, + IReadOnlyList fixtures, + string dtdRelative, + string dtdHash + ) + { + var text = new StringBuilder(); + text.Append(ConformanceManifest.ManifestFormat).Append('\n'); + text.Append(dtdRelative).Append('\0').Append(dtdHash).Append('\n'); + foreach (ManifestFixture fixture in fixtures) + { + text.Append(fixture.FixtureId) + .Append('\0') + .Append(fixture.GrammarSha256) + .Append('\0') + .Append(fixture.WordsSha256) + .Append('\n'); + } + + foreach (string relative in AdditionalSourceFiles.OrderBy(path => path, StringComparer.Ordinal)) + { + string absolute = Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)); + text.Append(relative) + .Append('\0') + .Append(File.Exists(absolute) ? HashFile(absolute) : "") + .Append('\n'); + } + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))).ToLowerInvariant(); + } + + /// SHA-256 of a file's text with line endings normalized to LF. + internal static string HashFile(string path) + { + string content = File.ReadAllText(path).Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(content))).ToLowerInvariant(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs new file mode 100644 index 000000000..f389ebf63 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs @@ -0,0 +1,22 @@ +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Builds the oracle for a fixture. The fixture's morpher: block is a +/// required argument, so no caller can run a fixture that overrides a setting at the defaults. +/// +internal static class ConformanceMorpherFactory +{ + internal static Morpher Create(Language language, MorpherSettings settings) + { + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + settings.ApplyTo(morpher); + return morpher; + } + + internal static Morpher CreateTracing(Language language, MorpherSettings settings) + { + var morpher = new Morpher(new TraceManager { IsTracing = true }, language, maxDegreeOfParallelism: 1); + settings.ApplyTo(morpher); + return morpher; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs new file mode 100644 index 000000000..0bdd76353 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Coverage and dead-rule measurement over the fixtures. +/// +/// blocked_by is asserted, not verified. A rule named in an expect_fail word's +/// blocked_by is credited as exercised, but nothing checks the trace to confirm that rule +/// actually blocked the word. The only check is that every label is a rule id declared in that +/// fixture's grammar (), which catches typos and stale ids, not +/// a label naming a real rule that never ran. Verifying labels against the parse trace is tracked +/// in sillsdev/machine#552. +/// +public static class CoverageReport +{ + public class DeadRule + { + public string FixtureId = ""; + public string Language = ""; + public string RuleId = ""; + } + + public class UnknownBlockedBy + { + public string FixtureId = ""; + public string Word = ""; + public string Label = ""; + } + + public class CoverageResult + { + public List DeadRules { get; } = new(); + + /// blocked_by labels that are not a rule id in their fixture's grammar. + public List UnknownBlockedBy { get; } = new(); + + /// Every construct string any word exercises (the distinct construct column + /// of coverage.csv). Diffed against constructs.txt by the caller to report absolute + /// construct coverage. + public HashSet CoveredConstructs { get; } = new(StringComparer.Ordinal); + } + + public static List LoadConstructChecklist(string path) + { + var constructs = new List(); + foreach (string rawLine in File.ReadAllLines(path)) + { + string line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith("#", StringComparison.Ordinal)) + continue; + constructs.Add(line); + } + return constructs; + } + + /// Reads dead-rules-allowed.tsv: one fixture-id TAB rule-id TAB reason row + /// per rule a fixture declares on purpose without any word exercising it. The reason column is + /// required so that no rule is allowed without saying why. + public static Dictionary<(string FixtureId, string RuleId), string> LoadDeadRuleAllowlist(string path) + { + var allowed = new Dictionary<(string, string), string>(); + int lineNumber = 0; + foreach (string rawLine in File.ReadAllLines(path)) + { + lineNumber++; + if (rawLine.Trim().Length == 0 || rawLine.StartsWith("#", StringComparison.Ordinal)) + continue; + string[] columns = rawLine.Split('\t'); + if (columns.Length != 3 || columns.Any(c => c.Trim().Length == 0)) + throw new FormatException($"{path}:{lineNumber}: expected fixture, rule and reason separated by tabs"); + if (!allowed.TryAdd((columns[0].Trim(), columns[1].Trim()), columns[2].Trim())) + throw new FormatException($"{path}:{lineNumber}: duplicate entry"); + } + return allowed; + } + + /// Splits the measured dead rules against the allowlist. Unexpected are dead rules + /// nobody has accepted; Stale are accepted rules that some word now exercises, whose entry + /// must go so the list cannot drift into accepting rules that are no longer dead. + public static (List Unexpected, List<(string FixtureId, string RuleId)> Stale) CheckDeadRules( + IReadOnlyCollection deadRules, + IReadOnlyDictionary<(string FixtureId, string RuleId), string> allowed + ) + { + List unexpected = deadRules.Where(d => !allowed.ContainsKey((d.FixtureId, d.RuleId))).ToList(); + var dead = new HashSet<(string, string)>(deadRules.Select(d => (d.FixtureId, d.RuleId))); + List<(string, string)> stale = allowed + .Keys.Where(k => !dead.Contains(k)) + .OrderBy(k => k.FixtureId, StringComparer.Ordinal) + .ThenBy(k => k.RuleId, StringComparer.Ordinal) + .ToList(); + return (unexpected, stale); + } + + /// Returns every blocked_by label, on an expect_fail or expect_skip word, that is not + /// in (the fixture grammar's declared rule ids). Ordinal comparison. + public static List FindUnknownBlockedBy( + string fixtureId, + WordsYaml words, + IReadOnlyCollection ruleIds + ) + { + var known = new HashSet(ruleIds, StringComparer.Ordinal); + var unknown = new List(); + foreach (WordEntry word in words.Words) + { + if (!word.ExpectFail && !word.ExpectSkip) + continue; + foreach (string label in word.BlockedBy) + { + if (!known.Contains(label)) + { + unknown.Add( + new UnknownBlockedBy + { + FixtureId = fixtureId, + Word = word.Word, + Label = label, + } + ); + } + } + } + return unknown; + } + + public static CoverageResult WriteCsvs(List fixtures, string coverageCsvPath, string rulesCsvPath) + { + var result = new CoverageResult(); + var coverageRows = new List<(string language, string word, string signature, string construct)>(); + var ruleRows = new List<(string language, string ruleId, string words)>(); + + foreach (Fixture fixture in fixtures) + { + string language = fixture.Words.Language; + GrammarRuleIndex ruleIndex = GrammarRuleIndex.Load(fixture.GrammarPath); + result.UnknownBlockedBy.AddRange(FindUnknownBlockedBy(fixture.Id, fixture.Words, ruleIndex.AllRuleIds)); + + // rule id -> exercising word tokens ("word" for a positive rules: hit, "!word" for a + // blocked_by attribution on an expect_fail word). + var exercisingWords = new Dictionary>(StringComparer.Ordinal); + void AddExercise(string ruleId, string token) + { + if (!exercisingWords.TryGetValue(ruleId, out List list)) + { + list = new List(); + exercisingWords[ruleId] = list; + } + list.Add(token); + } + + foreach (WordEntry word in fixture.Words.Words) + { + if (word.ExpectFail || word.ExpectSkip) + { + foreach (string construct in word.Exercises) + { + coverageRows.Add((language, word.Word, "", construct)); + result.CoveredConstructs.Add(construct); + } + foreach (string ruleId in word.BlockedBy) + AddExercise(ruleId, "!" + word.Word); + } + else + { + foreach (ParseEntry parse in word.Parses) + { + IEnumerable constructs = parse + .Exercises.Concat(word.Exercises) + .Distinct(StringComparer.Ordinal); + foreach (string construct in constructs) + { + coverageRows.Add((language, word.Word, parse.Signature, construct)); + result.CoveredConstructs.Add(construct); + } + foreach (string ruleId in parse.Rules) + AddExercise(ruleId, word.Word); + } + } + } + + foreach ( + string ruleId in ruleIndex + .AllRuleIds.Distinct(StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + ) + { + if (exercisingWords.TryGetValue(ruleId, out List words)) + { + ruleRows.Add((language, ruleId, string.Join(";", words.Distinct(StringComparer.Ordinal)))); + } + else + { + ruleRows.Add((language, ruleId, "")); + result.DeadRules.Add( + new DeadRule + { + FixtureId = fixture.Id, + Language = language, + RuleId = ruleId, + } + ); + } + } + } + + WriteCsv( + coverageCsvPath, + new[] { "language", "word", "signature", "construct" }, + coverageRows.Select(r => new[] { r.language, r.word, r.signature, r.construct }) + ); + WriteCsv( + rulesCsvPath, + new[] { "language", "rule_id", "words" }, + ruleRows.Select(r => new[] { r.language, r.ruleId, r.words }) + ); + + return result; + } + + private static void WriteCsv(string path, string[] header, IEnumerable rows) + { + // Always \n, so the checked-in CSVs compare clean on every platform (local_check.sh diffs them). + var sb = new StringBuilder(); + sb.Append(string.Join(",", header.Select(CsvField))).Append('\n'); + foreach (string[] row in rows) + sb.Append(string.Join(",", row.Select(CsvField))).Append('\n'); + File.WriteAllText(path, sb.ToString()); + } + + private static string CsvField(string value) + { + value ??= ""; + if (value.IndexOfAny(new[] { ',', '"', '\n', '\r' }) >= 0) + return "\"" + value.Replace("\"", "\"\"") + "\""; + return value; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs new file mode 100644 index 000000000..a568f325e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Per-word outcome of diffing one fixture's actual output against its expected.tsv. +public class WordResult +{ + public string Word = ""; + public bool Passed; + public string Detail = ""; +} + +public class FixtureDiff +{ + public List WordResults { get; } = new(); + public bool AllPassed => WordResults.All(w => w.Passed); +} + +public static class Diff +{ + public static FixtureDiff Compare(List expected, List actual) + { + var diff = new FixtureDiff(); + Dictionary> expByWord = GroupByWord(expected); + Dictionary> actByWord = GroupByWord(actual); + + IEnumerable allWords = expByWord + .Keys.Union(actByWord.Keys, StringComparer.Ordinal) + .OrderBy(w => w, StringComparer.Ordinal); + + foreach (string word in allWords) + { + bool hasExpected = expByWord.TryGetValue(word, out List expRows); + bool hasActual = actByWord.TryGetValue(word, out List actRows); + + if (!hasExpected) + { + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = false, + Detail = "present in adapter output but not in expected.tsv", + } + ); + continue; + } + if (!hasActual) + { + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = false, + Detail = "missing from adapter/self-check output", + } + ); + continue; + } + + List expCanon = Canonicalize(expRows); + List actCanon = Canonicalize(actRows); + bool passed = expCanon.SequenceEqual(actCanon, StringComparer.Ordinal); + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = passed, + Detail = passed + ? "ok" + : $"expected [{string.Join(" | ", expCanon)}] got [{string.Join(" | ", actCanon)}]", + } + ); + } + + return diff; + } + + private static Dictionary> GroupByWord(List rows) + { + var byWord = new Dictionary>(); + foreach (TsvRow row in rows) + { + if (!byWord.TryGetValue(row.Word, out List list)) + { + list = new List(); + byWord[row.Word] = list; + } + list.Add(row); + } + return byWord; + } + + // One canonical "status|sorted-signature-entries" string per row, sorted across the word's + // rows (usually just one) so occurrence order/duplication is handled as a multiset. + private static List Canonicalize(List rows) + { + return rows.Select(r => r.Status + "|" + string.Join(";", SignatureTsv.SplitComparableSignature(r.Signature))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs new file mode 100644 index 000000000..604d3941c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs @@ -0,0 +1,21 @@ +using System; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Thrown by an when the ENGINE UNDER TEST itself failed while loading a +/// grammar or parsing a word -- a genuine engine crash, as opposed to a harness-level failure +/// (the adapter process couldn't be started, timed out, or exited 0 without producing an output +/// file; see ). treats this exception specially: a +/// fixture whose manifest declares expectCrash: true (see ) +/// PASSes when this is thrown instead of the normal signature diff; any other fixture FAILs, +/// showing the carried diagnostics. +/// +public class EngineCrashException : Exception +{ + public EngineCrashException(string message) + : base(message) { } + + public EngineCrashException(string message, Exception innerException) + : base(message, innerException) { } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets new file mode 100644 index 000000000..7909286e2 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets @@ -0,0 +1,254 @@ + + + + + + + + + + + + + + + + > git = (workingDirectory, arguments) => + { + var startInfo = new ProcessStartInfo("git", arguments) + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.EnvironmentVariables["GIT_TERMINAL_PROMPT"] = "0"; + startInfo.EnvironmentVariables["GIT_OPTIONAL_LOCKS"] = "0"; + startInfo.EnvironmentVariables.Remove("GIT_DIR"); + startInfo.EnvironmentVariables.Remove("GIT_WORK_TREE"); + startInfo.EnvironmentVariables.Remove("GIT_INDEX_FILE"); + try + { + using (Process process = Process.Start(startInfo)) + { + var output = new StringBuilder(); + var error = new StringBuilder(); + process.OutputDataReceived += (sender, e) => { if (e.Data != null) lock (output) output.AppendLine(e.Data); }; + process.ErrorDataReceived += (sender, e) => { if (e.Data != null) lock (error) error.AppendLine(e.Data); }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + int limit = (int)Math.Max(10000, (deadline - DateTime.UtcNow).TotalMilliseconds); + if (!process.WaitForExit(limit)) + { + try + { + var killTree = typeof(Process).GetMethod("Kill", new[] { typeof(bool) }); + if (killTree != null) killTree.Invoke(process, new object[] { true }); + else process.Kill(); + } + catch (Exception) { } + process.WaitForExit(5000); + return new KeyValuePair(-1, "git did not finish in time: git " + arguments); + } + process.WaitForExit(); + return new KeyValuePair(process.ExitCode, process.ExitCode == 0 ? output.ToString() : error.ToString() + output.ToString()); + } + } + catch (Exception exception) + { + return new KeyValuePair(-1, "git could not be run: " + exception.Message); + } + }; + + Action deleteQuietly = path => + { + // Git writes its object files read-only, and Windows will not delete those. + try + { + foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + File.SetAttributes(file, FileAttributes.Normal); + Directory.Delete(path, true); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + }; + + try + { + // A temporary clone left by an interrupted build is never reused; remove the old ones. + if (Directory.Exists(parent)) + { + foreach (string stale in Directory.GetDirectories(parent, name + ".tmp-*")) + { + if (DateTime.UtcNow - Directory.GetCreationTimeUtc(stale) > TimeSpan.FromHours(1)) + deleteQuietly(stale); + } + } + + bool present = Directory.Exists(suite) && Directory.EnumerateFileSystemEntries(suite).Any(); + if (present) + { + if (!Directory.Exists(Path.Combine(suite, ".git")) && !File.Exists(Path.Combine(suite, ".git"))) + { + Log.LogError("conformance: " + suite + " exists but is not a git clone. Move it aside and build again, or set HC_CONFORMANCE_SUITE= to use it as it is."); + return false; + } + + KeyValuePair head = git(suite, "rev-parse HEAD"); + if (head.Key != 0) + { + Log.LogWarning("conformance: could not read the commit of " + suite + " (" + head.Value.Trim() + "). Run these two commands:" + movePin); + return true; + } + + KeyValuePair status = git(suite, "status --porcelain"); + string[] changed = status.Key == 0 + ? status.Value.Split('\n').Select(line => line.TrimEnd('\r')).Where(line => line.Length > 3).Select(line => line.Substring(3)).ToArray() + : new string[0]; + + if (head.Value.Trim() == pin) + { + // A suite author may be testing a change, so this is a warning and the build goes on. + if (changed.Length > 0) + Log.LogWarning("conformance: " + suite + " is at the pinned commit but has " + changed.Length + " local change(s): " + string.Join(", ", changed.Take(5)) + (changed.Length > 5 ? ", ..." : "") + ". The tests and the runner use what is on disk. Discard them unless you are testing a suite change."); + return true; + } + + if (changed.Length > 0) + { + Log.LogError("conformance: " + suite + " is at " + head.Value.Trim() + ", not the pinned " + pin + ", and has uncommitted changes (" + changed[0] + "), so it was left alone. Commit or discard them and build again, or set HC_CONFORMANCE_SUITE=" + suite + " to use it as it is."); + return false; + } + + if (git(suite, "cat-file -e " + pin + "^{commit}").Key != 0) + { + KeyValuePair fetch = git(suite, stalled + "fetch --quiet origin"); + if (fetch.Key != 0) + { + Log.LogWarning("conformance: " + suite + " is not at the pinned commit and fetching it failed (" + fetch.Value.Trim() + "). Run these two commands:" + movePin); + return true; + } + } + + KeyValuePair checkout = git(suite, stalled + "-c advice.detachedHead=false checkout --quiet --detach " + pin); + if (checkout.Key != 0) + Log.LogWarning("conformance: could not move " + suite + " to " + pin + " (" + checkout.Value.Trim() + "). Run this command: git -C " + name + " checkout --detach " + pin); + else + Log.LogMessage(MessageImportance.High, "conformance: moved " + suite + " from " + head.Value.Trim() + " to the pinned " + pin + "."); + return true; + } + + string temporary = Path.Combine(parent, name + ".tmp-" + Guid.NewGuid().ToString("N")); + Log.LogMessage(MessageImportance.High, "conformance: cloning " + SuiteUrl + " at " + pin + " into " + suite + "."); + KeyValuePair clone = git(parent, stalled + "clone --quiet --filter=blob:none --no-checkout " + SuiteUrl + " \"" + temporary + "\""); + if (clone.Key == 0) + clone = git(temporary, stalled + "-c advice.detachedHead=false checkout --quiet --detach " + pin); + if (clone.Key != 0) + { + deleteQuietly(temporary); + Log.LogWarning("conformance: could not clone the suite (" + clone.Value.Trim() + "). Run these two commands:" + clonePin); + return true; + } + + // Defender or the search indexer can hold a handle in the new tree for a moment, so retry the rename. + string constructs = Path.Combine(suite, "constructs.txt"); + string renameError = ""; + bool moved = false; + for (int attempt = 0; attempt < 5 && !moved; attempt++) + { + try + { + if (Directory.Exists(suite) && !Directory.EnumerateFileSystemEntries(suite).Any()) + Directory.Delete(suite); + Directory.Move(temporary, suite); + moved = true; + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException) + { + renameError = exception.Message; + if (File.Exists(constructs)) + break; + System.Threading.Thread.Sleep(500); + } + } + + if (!moved) + { + if (File.Exists(constructs)) + { + // A parallel build finished first; its clone stands if it is at the pin. + deleteQuietly(temporary); + KeyValuePair raced = git(suite, "rev-parse HEAD"); + if (raced.Key != 0 || raced.Value.Trim() != pin) + Log.LogWarning("conformance: " + suite + " appeared during the build and is not at " + pin + ". Run these two commands:" + movePin); + } + else + { + Log.LogWarning("conformance: cloned the suite but could not rename " + temporary + " to " + suite + " (" + renameError + "). The clone is complete: rename that directory to " + name + ", or build again."); + } + } + } + catch (Exception exception) + { + Log.LogWarning("conformance: could not make " + suite + " the suite at " + pin + " (" + exception.GetType().Name + ": " + exception.Message + "). Run these two commands:" + clonePin); + } + ]]> + + + + + + + + + diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs new file mode 100644 index 000000000..913cad0b7 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// One fixture directory: exactly grammar.xml + words.yaml, under +/// conformance/languages/<name>/ or conformance/edge-cases/<name>/. +/// +public class Fixture(string id, string directory, WordsYaml words) +{ + public string Id { get; } = id; + public string Directory { get; } = directory; + public WordsYaml Words { get; } = words; + + public string GrammarPath => Path.Combine(Directory, "grammar.xml"); + + /// The only directories under a fixtures root that hold fixtures. + public static readonly string[] Categories = { "languages", "edge-cases" }; + + public static List DiscoverAll(string fixturesRoot) + { + var fixtures = new List(); + foreach (string category in Categories) + { + string categoryRoot = Path.Combine(fixturesRoot, category); + if (!System.IO.Directory.Exists(categoryRoot)) + continue; + + foreach (string dir in System.IO.Directory.EnumerateDirectories(categoryRoot)) + { + string grammarPath = Path.Combine(dir, "grammar.xml"); + string wordsYamlPath = Path.Combine(dir, "words.yaml"); + if (!File.Exists(grammarPath) || !File.Exists(wordsYamlPath)) + continue; + + string name = Path.GetFileName(dir); + string id = $"{category}/{name}"; + WordsYaml words = WordsYamlLoader.Load(wordsYamlPath); + fixtures.Add(new Fixture(id, dir, words)); + } + } + fixtures.Sort((a, b) => string.CompareOrdinal(a.Id, b.Id)); + return fixtures; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs new file mode 100644 index 000000000..4ae50f35f --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// In-memory adapter-mode fixture descriptor, built by from a +/// fixture's grammar.xml + words.yaml. It is never read from or written to disk. +/// +public class FixtureManifest +{ + public string Id { get; set; } = ""; + + public string Category { get; set; } = "single-feature"; + + public List Requires { get; set; } = new(); + + /// The fixture's morpher: block; an engine runs this fixture only if it declares + /// , and must apply every value. + public MorpherSettings Morpher { get; set; } = new(); + + public bool ExpectCrash { get; set; } + + public FixtureBudget Budget { get; set; } + + public string Provenance { get; set; } = ""; +} + +/// +/// A pathological fixture's complexity budget: a wall-clock bound only, with no step-count ceiling. +/// Absent/null for non-pathological fixtures. +/// +public class FixtureBudget +{ + public long WallClockMs { get; set; } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs new file mode 100644 index 000000000..04e81e10c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs @@ -0,0 +1,126 @@ +using System; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Materializes a fixture (grammar.xml + words.yaml) into the on-disk adapter shape +/// (grammar.xml + words.txt + expected.tsv) in a temp directory, plus an in-memory +/// , so it can be run through + +/// for adapter mode without either needing to know about the +/// words.yaml format. This is the only code path that touches AdapterEngine/MaterializedRunner. +/// +public static class FixtureMaterializer +{ + /// + /// Builds the temp directory and returns a pointing at it. + /// Caller owns cleanup (delete the returned fixture's + /// when done). + /// + public static MaterializedFixture Materialize(Fixture fixture) + { + return Materialize(fixture, new WordRun("default", fixture.Words.Morpher, fixture.Words.Words)); + } + + public static MaterializedFixture Materialize(Fixture fixture, WordRun run) + { + var adapterWords = run.Morpher.GuessRoots.HasValue + ? run.Words + : run.Words.Where(w => w.Parses.All(p => !p.Guess)).ToList(); + if (run.Name != "default" && adapterWords.Count == 0) + { + throw new NamedVariantHasNoAdapterVisibleWordsException( + fixture.Id, + run.Name, + "no adapter-visible words remain after legacy guess filtering; set morpher.GuessRoots explicitly " + + "if the variant is intended to include guessed-word rows" + ); + } + + string tempDir = Path.Combine(Path.GetTempPath(), "hc-conformance-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + File.Copy(fixture.GrammarPath, Path.Combine(tempDir, "grammar.xml")); + + string wordsPath = Path.Combine(tempDir, "words.txt"); + File.WriteAllLines(wordsPath, adapterWords.Select(w => w.Word)); + + string expectedPath = Path.Combine(tempDir, "expected.tsv"); + using (var writer = new StreamWriter(expectedPath)) + { + if (fixture.Words.ExpectCrash) + { + WordEntry first = fixture.Words.Words[0]; + writer.WriteLine($"0\t{first.Word}\tSTARTED"); + } + else + { + int i = 0; + foreach (WordEntry word in adapterWords) + { + string signature = BuildExpectedSignature(word); + // expect_skip words carry status "SKIPPED" (the oracle throws InvalidShapeException), + // matching the expected.tsv row shape "N word 0 SKIPPED -"; everything else is "ok". + string status = word.ExpectSkip ? "SKIPPED" : "ok"; + writer.WriteLine($"{i}\t{word.Word}\t0\t{status}\t{signature}"); + i++; + } + } + } + + File.WriteAllText( + Path.Combine(tempDir, MaterializedFixture.MorpherFileName), + System.Text.Json.JsonSerializer.Serialize(run.Morpher.ToDictionary()) + ); + + var manifest = new FixtureManifest + { + Id = fixture.Id, + Category = fixture.Words.BudgetMs.HasValue ? "pathological" : "single-feature", + Requires = fixture.Words.Requires, + Morpher = run.Morpher, + ExpectCrash = fixture.Words.ExpectCrash, + Budget = fixture.Words.BudgetMs.HasValue + ? new FixtureBudget { WallClockMs = fixture.Words.BudgetMs.Value } + : null, + Provenance = string.Join( + "; ", + run.Words.Where(w => !string.IsNullOrEmpty(w.Provenance)).Select(w => w.Provenance) + ), + }; + + bool requiresMorpherSettings = RequiresDerivation + .Derive(fixture.GrammarPath, fixture.Words.Runs.Select(candidate => candidate.Morpher)) + .Contains(MorpherSettings.Capability, StringComparer.Ordinal); + return new MaterializedFixture( + tempDir, + manifest, + run.Name == "default" ? null : run.Name, + requiresMorpherSettings + ); + } + + public static void Cleanup(MaterializedFixture materialized) + { + try + { + if (Directory.Exists(materialized.Directory)) + Directory.Delete(materialized.Directory, recursive: true); + } + catch (IOException) { } + } + + /// The signature string materialized into expected.tsv column 5: "-" for a word with zero expected + /// parses (expect_fail), else the ";"-joined, ordinally-sorted set of parses[].signature -- the same shape + /// BatchCommand.BuildSignature itself produces. + public static string BuildExpectedSignature(WordEntry word) + { + if (word.ExpectFail || word.ExpectSkip || word.Parses.Count == 0) + return "-"; + return string.Join(";", word.Parses.Select(p => p.Signature).OrderBy(s => s, StringComparer.Ordinal)); + } +} + +internal sealed class NamedVariantHasNoAdapterVisibleWordsException(string fixtureId, string variantName, string reason) + : InvalidOperationException($"fixture '{fixtureId}' variant '{variantName}' {reason}.") { } diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs new file mode 100644 index 000000000..05e9efd60 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml; +using System.Xml.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Independently re-parses a fixture's grammar.xml (not via , +/// whose runtime rule objects lose the XML id attribute for several rule kinds -- see below) +/// to answer two questions the harness needs for the "rules:" coverage dimension: (1) every rule id +/// the grammar declares (for dead-rule detection), and (2) how to translate a *runtime* rule +/// identity (what or a node's +/// Source actually expose at parse time) back to that XML id attribute, so a traced +/// rule application can be compared against a words.yaml "rules:" entry. +/// +/// Why this indirection is needed (see XmlLanguageLoader.TryLoadAffixProcessRule / +/// TryLoadCompoundingRule / LoadRewriteRule / LoadMetathesisRule): the XML +/// id attribute on a MorphologicalRule/RealizationalRule element is used only +/// as the loader's own lookup key while building the graph -- the *runtime* +/// object's is loaded from the separate <MorphemeId> child +/// element instead (the short code that appears in a parse's morph-chain signature, e.g. "PL"). +/// A CompoundingRule, PhonologicalRule, and MetathesisRule element has no +/// MorphemeId at all; their runtime objects expose only (from the +/// element's <Name> child). None of the four runtime rule kinds retains the XML +/// id attribute itself anywhere the harness can read it back after loading. This class +/// rebuilds that lost mapping straight from the XML. +/// +public class GrammarRuleIndex +{ + /// Every rule id declared by the grammar (MorphologicalRule/RealizationalRule/CompoundingRule/ + /// PhonologicalRule/MetathesisRule ids, plus the co-occurrence-rule pseudo-ids below), in document + /// order. + public List AllRuleIds { get; } = new(); + + private readonly Dictionary _ruleIdByMorphemeId = new(StringComparer.Ordinal); + + // text -> XML id attribute, for CompoundingRule/PhonologicalRule/MetathesisRule: the only + // identity these three retain at runtime is IHCRule.Name. + private readonly Dictionary _ruleIdByRuleName = new(StringComparer.Ordinal); + + public HashSet CoOccurrencePseudoIds { get; } = new(StringComparer.Ordinal); + + public static GrammarRuleIndex Load(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + var index = new GrammarRuleIndex(); + foreach (XElement el in doc.Descendants()) + { + switch (el.Name.LocalName) + { + case "MorphologicalRule": + case "RealizationalRule": + { + string id = (string)el.Attribute("id"); + string morphemeId = (string)el.Element("MorphemeId"); + if (id == null) + continue; + index.AllRuleIds.Add(id); + if (morphemeId != null) + { + AddUnique(index._ruleIdByMorphemeId, morphemeId, id, grammarPath, "MorphemeId"); + } + else + { + string name = (string)el.Element("Name"); + if (name != null) + AddUnique(index._ruleIdByRuleName, name, id, grammarPath, "Name"); + } + break; + } + case "CompoundingRule": + case "PhonologicalRule": + case "MetathesisRule": + { + string id = (string)el.Attribute("id"); + string name = (string)el.Element("Name"); + if (id == null) + continue; + index.AllRuleIds.Add(id); + if (name != null) + AddUnique(index._ruleIdByRuleName, name, id, grammarPath, "Name"); + break; + } + case "MorphemeCoOccurrenceRule": + case "AllomorphCoOccurrenceRule": + { + string primary = (string)(el.Attribute("primaryMorpheme") ?? el.Attribute("primaryAllomorph")); + if (primary == null) + continue; + index.AllRuleIds.Add(primary); + index.CoOccurrencePseudoIds.Add(primary); + break; + } + } + } + return index; + } + + /// Records under , throwing if a different rule already + /// claimed that key: a fired rule is resolved by key, so a shared key would credit the wrong rule. + private static void AddUnique( + Dictionary map, + string key, + string id, + string grammarPath, + string keyKind + ) + { + if (map.TryGetValue(key, out string existing) && !string.Equals(existing, id, StringComparison.Ordinal)) + { + throw new InvalidDataException( + $"{grammarPath}: rules '{existing}' and '{id}' share the {keyKind} '{key}', so a fired rule cannot be attributed" + ); + } + map[key] = id; + } + + /// + /// Translates a fired (from , + /// or a RealizationalAffixProcessRule taken from a MorphologicalRuleSynthesis trace node -- see + /// ) back to its grammar.xml rule id. Returns null (rather than + /// throwing) for a rule kind this index cannot attribute. + /// + public string ResolveMorphologicalRuleId(IMorphologicalRule rule) + { + if ( + rule is Morpheme morpheme + && morpheme.Id != null + && _ruleIdByMorphemeId.TryGetValue(morpheme.Id, out string byMorphemeId) + ) + { + return byMorphemeId; + } + if (rule.Name != null && _ruleIdByRuleName.TryGetValue(rule.Name, out string byName)) + return byName; + return null; + } + + /// Translates a fired phonological rule (from a trace node's Source) back to its grammar.xml + /// rule id, by IHCRule.Name -- see the class doc comment. + public string ResolvePhonologicalRuleId(IPhonologicalRule rule) + { + return rule.Name != null && _ruleIdByRuleName.TryGetValue(rule.Name, out string id) ? id : null; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs new file mode 100644 index 000000000..bacd3dfbc --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs @@ -0,0 +1,75 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Validates a fixture grammar against the pinned DTD with no external reach. +/// The only admitted external subset is the exact system identifier +/// HermitCrabInput.dtd, remapped to the suite's canonical DTD whatever directory the +/// fixture lives in. Network access and filesystem fallback are refused rather than attempted. +public static class GrammarValidation +{ + public const string AdmittedSystemId = "HermitCrabInput.dtd"; + + /// Validates , returning every validation message. + public static IReadOnlyList Validate(string grammarPath, string dtdPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(grammarPath); + ArgumentException.ThrowIfNullOrWhiteSpace(dtdPath); + + var messages = new List(); + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Parse, + ValidationType = ValidationType.DTD, + XmlResolver = new PinnedDtdResolver(Path.GetFullPath(dtdPath)), + }; + settings.ValidationEventHandler += (_, args) => messages.Add($"{args.Severity}: {args.Message}"); + + // The document is opened here so the resolver is consulted only for the DTD, never for the + // grammar's own URI. + using var grammar = new FileStream(grammarPath, FileMode.Open, FileAccess.Read, FileShare.Read); + using XmlReader reader = XmlReader.Create( + grammar, + settings, + new Uri(Path.GetFullPath(grammarPath)).AbsoluteUri + ); + while (reader.Read()) + { + // Validation is reported through the event handler; the read loop only drives it. + } + + return messages; + } + + private sealed class PinnedDtdResolver(string dtdPath) : XmlResolver + { + private readonly string _dtdPath = dtdPath; + + // Called for the document's own base URI as well as for external subsets, so admission is + // decided in GetEntity, which is where content would actually be opened. + public override Uri ResolveUri(Uri? baseUri, string? relativeUri) => + string.Equals(relativeUri, AdmittedSystemId, StringComparison.Ordinal) + ? new Uri(_dtdPath) + : base.ResolveUri(baseUri, relativeUri); + + public override object GetEntity(Uri absoluteUri, string? role, Type? typeOfObjectToReturn) + { + ArgumentNullException.ThrowIfNull(absoluteUri); + if ( + !absoluteUri.IsFile + || !string.Equals(Path.GetFullPath(absoluteUri.LocalPath), _dtdPath, StringComparison.OrdinalIgnoreCase) + ) + { + throw new XmlException( + $"External entity '{absoluteUri}' is not admitted; only '{AdmittedSystemId}' may be referenced." + ); + } + + return new FileStream(_dtdPath, FileMode.Open, FileAccess.Read, FileShare.Read); + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs new file mode 100644 index 000000000..7fbeed32a --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public interface IEngine +{ + /// A short, human-readable name for reporting (e.g. "self-check", "adapter"). + string Name { get; } + + /// + /// The capability set this engine declares support for. Self-check mode always reports the + /// full set (it IS the reference C# oracle); an adapter's set comes from the --capabilities + /// flag. + /// + IReadOnlySet Capabilities { get; } + + /// Runs every word in the fixture's words.txt, returning the resulting TSV rows. + List Run(MaterializedFixture fixture); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs new file mode 100644 index 000000000..b22bb02c2 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs @@ -0,0 +1,38 @@ +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// A fixture materialized onto disk in the on-disk adapter shape (grammar.xml + words.txt + +/// expected.tsv + an in-memory ), so it can be handed to +/// and (self-check or an external adapter +/// process). Built by from a . +/// +public class MaterializedFixture( + string directory, + FixtureManifest manifest, + string variantName = null, + bool? requiresMorpherSettings = null +) +{ + public string Id { get; } = variantName == null ? manifest.Id : $"{manifest.Id} [variant {variantName}]"; + public string Directory { get; } = directory; + public FixtureManifest Manifest { get; } = manifest; + + /// + /// Fixture-level capability requirement. This can be true for the default run even when its own + /// settings are empty, because one of the fixture's named variants declares settings. + /// + public bool RequiresMorpherSettings { get; } = requiresMorpherSettings ?? !manifest.Morpher.IsEmpty; + + public string GrammarPath => Path.Combine(Directory, "grammar.xml"); + public string WordsPath => Path.Combine(Directory, "words.txt"); + public string ExpectedPath => Path.Combine(Directory, "expected.tsv"); + + /// + /// The JSON object of this run's morpher: block, keyed by FieldWorks element name; {} when absent. + /// + public const string MorpherFileName = "morpher.json"; + + public string MorpherPath => Path.Combine(Directory, MorpherFileName); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs new file mode 100644 index 000000000..98a9549ab --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public enum FixtureOutcome +{ + Passed, + Failed, + Skipped, +} + +public class FixtureResult +{ + public string FixtureId = ""; + public FixtureOutcome Outcome; + public string Reason = ""; + public long ElapsedMs; + public List WordResults = new(); +} + +public class MaterializedRunReport +{ + public List Results { get; } = new(); + public int ExcludedPathologicalCount { get; set; } + + public int Passed => Results.Count(r => r.Outcome == FixtureOutcome.Passed); + public int Failed => Results.Count(r => r.Outcome == FixtureOutcome.Failed); + public int Skipped => Results.Count(r => r.Outcome == FixtureOutcome.Skipped); +} + +/// +/// Runs a list of s through an , applying +/// category exclusion (pathological) and capability filtering (manifest "requires" vs. the engine's +/// declared capabilities), and diffs results against each fixture's expected.tsv. Called by +/// against fixtures writes to disk. +/// +public static class MaterializedRunner +{ + public static MaterializedRunReport Run( + List fixtures, + IEngine engine, + bool includePathological + ) + { + var report = new MaterializedRunReport(); + + foreach (MaterializedFixture fixture in fixtures) + { + var result = new FixtureResult { FixtureId = fixture.Id }; + if (IsExcludedPathological(fixture, includePathological)) + { + report.ExcludedPathologicalCount++; + continue; + } + + List derivedRequires = DerivedRequires(fixture); + if (!RequiresMatchGrammar(fixture, derivedRequires)) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"manifest error: requires [{string.Join(",", fixture.Manifest.Requires)}] does not match " + + $"grammar-derived [{string.Join(",", derivedRequires)}]"; + report.Results.Add(result); + continue; + } + + List missingCapabilities = MissingCapabilities(fixture, engine); + if (missingCapabilities.Count > 0) + { + result.Outcome = FixtureOutcome.Skipped; + result.Reason = + $"requires [{string.Join(",", fixture.Manifest.Requires)}], " + + $"engine declares [{string.Join(",", engine.Capabilities.OrderBy(c => c))}]; " + + $"missing [{string.Join(",", missingCapabilities)}]"; + report.Results.Add(result); + continue; + } + + // Read once, before the try, so a missing expected.tsv cannot throw from inside a catch block. + List expected = SignatureTsv.ReadFile(fixture.ExpectedPath); + + var sw = Stopwatch.StartNew(); + try + { + List actual = RunWithBudget(engine, fixture); + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Manifest.ExpectCrash) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = "expected crash, engine produced output"; + } + else + { + FixtureDiff diff = Diff.Compare(expected, actual); + result.WordResults = diff.WordResults; + + // Enforced whenever a budget is present, regardless of category: today only + // pathological fixtures carry one, but nothing about the check is category- + // specific. + bool overBudget = + fixture.Manifest.Budget != null && result.ElapsedMs > fixture.Manifest.Budget.WallClockMs; + + if (overBudget) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"exceeded wall-clock budget ({result.ElapsedMs}ms > {fixture.Manifest.Budget.WallClockMs}ms)"; + } + else if (diff.AllPassed) + { + result.Outcome = FixtureOutcome.Passed; + } + else + { + result.Outcome = FixtureOutcome.Failed; + // Name the words. A bare count cannot be diagnosed from a CI log on a machine you cannot reach. + + WordResult[] mismatched = diff.WordResults.Where(word => !word.Passed).ToArray(); + + string named = string.Join( + "; ", + mismatched.Take(5).Select(word => $"{word.Word}: {Truncate(word.Detail)}") + ); + + result.Reason = + mismatched.Length > 5 + ? $"{mismatched.Length}/{ +diff.WordResults +.Count} word(s) mismatched: {named}; and {mismatched.Length - 5} more" + : $"{mismatched.Length}/{ +diff.WordResults +.Count} word(s) mismatched: {named}"; + } + } + } + catch (EngineCrashException ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Manifest.ExpectCrash) + { + // A misconfigured adapter exits nonzero on every fixture, so credit the crash only + // if this engine also completes an ordinary fixture this run would execute. + MaterializedFixture control = fixtures.FirstOrDefault(candidate => + !candidate.Manifest.ExpectCrash && WouldRun(candidate, engine, includePathological) + ); + bool adapterIsFunctional = control == null || EngineCanRunControl(engine, control); + if (adapterIsFunctional) + { + result.Outcome = FixtureOutcome.Passed; + result.Reason = $"engine crashed as expected (manifest expectCrash: true): {ex.Message}"; + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"engine crashed, but also failed the control fixture '{control.Id}' -- " + + $"treating as a harness-level problem, not a crash match: {ex.Message}"; + } + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"engine crashed: {ex.Message}"; + } + } + catch (TimeoutException ex) when (fixture.Manifest.Budget != null) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"exceeded budget {fixture.Manifest.Budget.WallClockMs}ms: {ex.Message}"; + } + catch (Exception ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + // Any other exception is a plain harness error (bad --adapter template, process + // couldn't start, timed out, etc.) -- never a pass, regardless of expectCrash. + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"harness error: {ex.Message}"; + } + + report.Results.Add(result); + } + + return report; + } + + /// + /// Runs the engine for one fixture. If the manifest carries a budget, engine.Run happens on a + /// background thread and this waits up to budget + a small margin; AdapterEngine independently + /// plumbs the same budget into its own process timeout (so a subprocess adapter gets killed), + /// but self-check's in-process call has no external process to kill, so this watchdog is what + /// keeps a runaway self-check parse from hanging the whole harness run. If the watchdog expires, + /// a TimeoutException is thrown and the background thread is simply abandoned -- this is a + /// short-lived CLI, so a leaked runaway thread on the rare budget-blown fixture is an acceptable + /// trade for keeping this shape simple. + /// + private static List RunWithBudget(IEngine engine, MaterializedFixture fixture) + { + FixtureBudget budget = fixture.Manifest.Budget; + if (budget == null) + return engine.Run(fixture); + + Task> task = Task.Run(() => engine.Run(fixture)); + long marginMs = Math.Max(budget.WallClockMs / 5, 1000); + bool completed; + try + { + completed = task.Wait((int)(budget.WallClockMs + marginMs)); + } + catch (AggregateException ex) when (ex.InnerException != null) + { + // Unwrap: rethrow the engine's own exception so the caller's EngineCrashException/ + // generic catch blocks see the real exception type, not Task's wrapper. + throw ex.InnerException; + } + if (!completed) + throw new TimeoutException("watchdog timed out waiting for engine.Run"); + return task.GetAwaiter().GetResult(); + } + + private static bool IsExcludedPathological(MaterializedFixture fixture, bool includePathological) => + !includePathological && string.Equals(fixture.Manifest.Category, "pathological", StringComparison.Ordinal); + + private static List DerivedRequires(MaterializedFixture fixture) => + RequiresDerivation.Derive(fixture.GrammarPath, fixture.RequiresMorpherSettings); + + private static bool RequiresMatchGrammar(MaterializedFixture fixture, List derivedRequires) => + new HashSet(derivedRequires, StringComparer.Ordinal).SetEquals( + new HashSet(fixture.Manifest.Requires, StringComparer.Ordinal) + ); + + private static List MissingCapabilities(MaterializedFixture fixture, IEngine engine) => + fixture.Manifest.Requires.Where(req => !engine.Capabilities.Contains(req)).ToList(); + + /// + /// Whether would execute this fixture rather than exclude it, report a manifest + /// error, or skip it for a missing capability. + /// + private static bool WouldRun(MaterializedFixture fixture, IEngine engine, bool includePathological) => + !IsExcludedPathological(fixture, includePathological) + && RequiresMatchGrammar(fixture, DerivedRequires(fixture)) + && MissingCapabilities(fixture, engine).Count == 0; + + /// + /// Whether this exact engine can still complete an ordinary fixture, run fresh (not read back + /// from an earlier result) so the check is independent of run order or a single-fixture run. Any + /// exception -- crash, timeout, or otherwise -- means no: only a normal return counts. + /// + private static bool EngineCanRunControl(IEngine engine, MaterializedFixture control) + { + try + { + RunWithBudget(engine, control); + return true; + } + catch + { + return false; + } + } + + /// Shortens a per-word detail so a fixture reason stays readable in a log. + /// The word detail, which may carry every expected and actual parse. + private static string Truncate(string detail) + { + const int Limit = 160; + if (string.IsNullOrEmpty(detail) || detail.Length <= Limit) + { + return detail; + } + + return detail[..Limit] + "..."; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs new file mode 100644 index 000000000..fbe202e1c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// A fixture's morpher: front-matter block: host settings the run's +/// ground truth was generated under, keyed by the FieldWorks <ParserParameters><HC> +/// element names that set them (HCParser.LoadParser). Every property is null when the block +/// omits it, which means the constructor default. Only settings whose effect +/// is a parse result are accepted; anything else is a +/// error. +/// +public class MorpherSettings +{ + /// The requires capability a fixture with a non-empty block must declare. + public const string Capability = "morpher-settings"; + + /// + /// No overrides: every property at its constructor default. Named so a caller + /// that deliberately ignores a fixture's block says so. + /// + public static MorpherSettings Defaults => new(); + + /// DelReapps: , at least 0. + public int? DelReapps { get; set; } + + /// MaxRoots: , at least 1. + public int? MaxRoots { get; set; } + + /// MergeAnalyses: . + public bool? MergeAnalyses { get; set; } + + /// MakePartialsFinal: . + public bool? MakePartialsFinal { get; set; } + + /// Per-call ParseWord guessRoot flag; this is not a Morpher property. + public bool? GuessRoots { get; set; } + + public bool IsEmpty => + DelReapps == null + && MaxRoots == null + && MergeAnalyses == null + && MakePartialsFinal == null + && GuessRoots == null; + + /// + /// Resolve the per-call root-guess flag for a word, preserving legacy per-word behavior unless + /// this run explicitly supplies GuessRoots. + /// + public bool GuessRootsFor(WordEntry word) + { + return GuessRoots ?? word.Parses.Any(parse => parse.Guess); + } + + /// Sets every declared value on , leaving the rest at their defaults. + public void ApplyTo(Morpher morpher) + { + if (DelReapps is int delReapps) + morpher.DeletionReapplications = delReapps; + if (MaxRoots is int maxRoots) + morpher.MaxStemCount = maxRoots; + if (MergeAnalyses is bool mergeAnalyses) + morpher.MergeEquivalentAnalyses = mergeAnalyses; + if (MakePartialsFinal is bool makePartialsFinal) + morpher.AlwaysEnforceFinalTemplates = makePartialsFinal; + } + + /// The declared values by FieldWorks element name, for manifest.json and adapters. + public SortedDictionary ToDictionary() + { + var values = new SortedDictionary(System.StringComparer.Ordinal); + if (DelReapps is int delReapps) + values["DelReapps"] = delReapps; + if (MaxRoots is int maxRoots) + values["MaxRoots"] = maxRoots; + if (MergeAnalyses is bool mergeAnalyses) + values["MergeAnalyses"] = mergeAnalyses; + if (MakePartialsFinal is bool makePartialsFinal) + values["MakePartialsFinal"] = makePartialsFinal; + if (GuessRoots is bool guessRoots) + values["GuessRoots"] = guessRoots; + return values; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs new file mode 100644 index 000000000..98dc5054e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs @@ -0,0 +1,445 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +internal class Program +{ + private static int Main(string[] args) + { + string fixturesRoot = null; + string adapterTemplate = null; + string capabilitiesArg = null; + bool capabilitiesProvided = false; + bool includePathological = false; + bool coverageReport = false; + string constructsPath = null; + string coverageOutput = null; + bool propose = false; + bool checkManifest = false; + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--fixtures": + if (!TryGetNextArg(args, ref i, "--fixtures", out fixturesRoot)) + return 2; + break; + case "--adapter": + if (!TryGetNextArg(args, ref i, "--adapter", out adapterTemplate)) + return 2; + break; + case "--capabilities": + if (!TryGetNextArg(args, ref i, "--capabilities", out capabilitiesArg)) + return 2; + capabilitiesProvided = true; + break; + case "--include-pathological": + includePathological = true; + break; + case "--coverage-report": + coverageReport = true; + break; + case "--constructs": + if (!TryGetNextArg(args, ref i, "--constructs", out constructsPath)) + return 2; + break; + case "--coverage-output": + if (!TryGetNextArg(args, ref i, "--coverage-output", out coverageOutput)) + return 2; + break; + case "--propose": + propose = true; + break; + case "--check-manifest": + checkManifest = true; + break; + case "-h": + case "--help": + PrintUsage(); + return 0; + default: + Console.Error.WriteLine($"unrecognized argument: {args[i]}"); + PrintUsage(); + return 2; + } + } + + if (string.IsNullOrEmpty(fixturesRoot)) + { + Console.Error.WriteLine("--fixtures is required"); + PrintUsage(); + return 2; + } + + foreach (string warning in SuitePinCheck.Warnings(fixturesRoot)) + Console.Error.WriteLine($"warning: {warning}"); + + List fixtures; + try + { + fixtures = Fixture.DiscoverAll(fixturesRoot); + } + catch (WordsYamlException ex) + { + Console.Error.WriteLine($"words.yaml error: {ex.Message}"); + return 2; + } + Console.WriteLine($"discovered {fixtures.Count} fixture(s) under '{fixturesRoot}'"); + + if (fixtures.Count == 0) + { + Console.Error.WriteLine( + $"no fixtures discovered under '{fixturesRoot}'; if this is the conformance/ clone of the suite, " + + "run: dotnet build src/SIL.Machine.Morphology.HermitCrab.Conformance" + ); + return 2; + } + + if (checkManifest) + return RunCheckManifest(fixturesRoot); + + if (coverageReport) + { + constructsPath ??= Path.Combine(fixturesRoot, "constructs.txt"); + return RunCoverageReport(fixtures, fixturesRoot, constructsPath, coverageOutput); + } + + IEngine engine; + if (adapterTemplate != null) + { + HashSet capabilities = ParseCapabilities(capabilitiesArg ?? ""); + engine = new AdapterEngine(adapterTemplate, capabilities); + } + else + { + IReadOnlySet capabilities = capabilitiesProvided ? ParseCapabilities(capabilitiesArg) : null; + engine = new SelfCheckEngine(capabilities); + } + + RunReport report = + adapterTemplate != null + ? Runner.RunAdapter(fixtures, engine, includePathological) + : Runner.RunSelfCheck(fixtures, includePathological, engine.Capabilities, propose, Console.Out); + PrintRunReport(report, engine); + + bool anyRan = report.Passed > 0 || report.Failed > 0; + if (!anyRan) + { + Console.Error.WriteLine("no fixtures actually ran (all excluded or skipped) -- treating this as an error"); + return 2; + } + + return report.Failed > 0 ? 1 : 0; + } + + /// Returns the argument following , or prints usage and returns + /// false if was the last token (no bounds-check crash on a truncated + /// command line). + private static bool TryGetNextArg(string[] args, ref int i, string flag, out string value) + { + if (i + 1 >= args.Length) + { + Console.Error.WriteLine($"{flag} requires a value"); + PrintUsage(); + value = null; + return false; + } + value = args[++i]; + return true; + } + + private static HashSet ParseCapabilities(string arg) + { + return arg.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.Ordinal); + } + + private static void PrintRunReport(RunReport report, IEngine engine) + { + Console.WriteLine(); + Console.WriteLine( + $"engine: {engine.Name}, capabilities: [{string.Join(",", engine.Capabilities.OrderBy(c => c))}]" + ); + if (report.ExcludedPathologicalCount > 0) + { + Console.WriteLine( + $"{report.ExcludedPathologicalCount} pathological run(s) (budget_ms) excluded by default (--include-pathological to run)" + ); + } + Console.WriteLine(); + + foreach (FixtureResult result in report.Results) + { + string status = result.Outcome switch + { + FixtureOutcome.Passed => "PASS", + FixtureOutcome.Failed => "FAIL", + FixtureOutcome.Skipped => "SKIP", + _ => "?", + }; + Console.WriteLine($"[{status}] {result.FixtureId} ({result.ElapsedMs}ms) {result.Reason}"); + if (result.Outcome == FixtureOutcome.Failed) + { + foreach (WordResult w in result.WordResults.Where(w => !w.Passed)) + Console.WriteLine($" word '{w.Word}': {w.Detail}"); + } + } + + Console.WriteLine(); + Console.WriteLine( + $"totals: {report.Passed} runs passed, {report.Failed} failed, {report.Skipped} skipped " + + $"(of {report.Results.Count} runs attempted across {report.FixtureCount} fixtures)" + ); + } + + /// Verifies the suite's checked-in fixture manifest byte for byte against a fresh computation, after + /// validating every grammar against the DTD and every words.yaml against its schema. The suite owns the + /// manifest and writes it with its own generator, so this never writes. + private static int RunCheckManifest(string suiteRoot) + { + string relative = ConformanceManifestGenerator.ManifestRelativePath; + string path = Path.Combine(suiteRoot, relative.Replace('/', Path.DirectorySeparatorChar)); + string dtd = Path.Combine(suiteRoot, ConformanceManifestGenerator.DtdRelativePath); + string wordsSchema = Path.Combine( + suiteRoot, + WordsSchemaValidation.SchemaRelativePath.Replace('/', Path.DirectorySeparatorChar) + ); + + ConformanceManifest corpus; + try + { + corpus = ConformanceManifestGenerator.Generate(suiteRoot); + } + catch (Exception ex) + when (ex is IOException or UnauthorizedAccessException or InvalidDataException or WordsYamlException) + { + Console.Error.WriteLine($"cannot compute the manifest under '{suiteRoot}': {ex.Message}"); + return 2; + } + + // Validate every fixture before comparing, so a rejected one is named rather than reported as stale. + foreach (ManifestFixture fixture in corpus.Fixtures) + { + IReadOnlyList messages = GrammarValidation.Validate( + Path.Combine(suiteRoot, fixture.GrammarPath.Replace('/', Path.DirectorySeparatorChar)), + dtd + ); + if (messages.Count != 0) + { + Console.Error.WriteLine($"{fixture.FixtureId}: grammar does not validate against {corpus.DtdPath}"); + foreach (string message in messages) + Console.Error.WriteLine($" {message}"); + return 2; + } + + messages = WordsSchemaValidation.Validate( + Path.Combine(suiteRoot, fixture.WordsPath.Replace('/', Path.DirectorySeparatorChar)), + wordsSchema + ); + if (messages.Count == 0) + continue; + Console.Error.WriteLine( + $"{fixture.FixtureId}: words.yaml does not conform to {WordsSchemaValidation.SchemaRelativePath}" + ); + foreach (string message in messages) + Console.Error.WriteLine($" {message}"); + return 2; + } + + string json = ManifestJson.Serialize(corpus) + "\n"; + if (!File.Exists(path)) + { + Console.Error.WriteLine($"{path} is missing; regenerate it in the suite with scripts/generate_manifest.py"); + return 1; + } + + if (!string.Equals(File.ReadAllText(path).ReplaceLineEndings("\n"), json, StringComparison.Ordinal)) + { + Console.Error.WriteLine($"{path} is stale; regenerate it in the suite with scripts/generate_manifest.py"); + return 1; + } + + Console.WriteLine($"{path} is current ({corpus.Fixtures.Count} fixtures)"); + return 0; + } + + private const string OutOfScopeConstruct = "Tracing (TraceType)"; + + /// Recomputes coverage.csv and rules.csv without touching the fixtures root, then returns 1 if the + /// suite's checked-in copies are stale, an in-scope construct has no coverage, a blocked_by label is not a + /// grammar rule id, a rule no word exercises is missing from dead-rules-allowed.tsv, or an allowed rule + /// is no longer dead; 0 otherwise. The suite owns both CSVs, so they are written only into + /// , never into the fixtures root. + private static int RunCoverageReport( + List fixtures, + string fixturesRoot, + string constructsPath, + string coverageOutput + ) + { + bool failed = false; + string scratch = Path.Combine(Path.GetTempPath(), $"hc-coverage-report-{Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + CoverageReport.CoverageResult result; + try + { + string freshCoverage = Path.Combine(scratch, "coverage.csv"); + string freshRules = Path.Combine(scratch, "rules.csv"); + result = CoverageReport.WriteCsvs(fixtures, freshCoverage, freshRules); + + Console.WriteLine(); + Console.WriteLine("coverage report"); + Console.WriteLine("==============="); + foreach (string name in new[] { "coverage.csv", "rules.csv" }) + { + string checkedIn = Path.Combine(fixturesRoot, name); + bool current = + File.Exists(checkedIn) + && string.Equals( + File.ReadAllText(checkedIn).ReplaceLineEndings("\n"), + File.ReadAllText(Path.Combine(scratch, name)).ReplaceLineEndings("\n"), + StringComparison.Ordinal + ); + Console.WriteLine($"{checkedIn} is {(current ? "current" : "STALE or missing")}"); + failed |= !current; + } + + if (coverageOutput != null) + { + string fixturesFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(fixturesRoot)); + string outputFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(coverageOutput)); + if ( + outputFull.Equals(fixturesFull, StringComparison.Ordinal) + || outputFull.StartsWith(fixturesFull + Path.DirectorySeparatorChar, StringComparison.Ordinal) + ) + { + Console.Error.WriteLine( + $"--coverage-output {coverageOutput} is inside the fixtures root; write into a separate clone of the suite, not the suite clone" + ); + return 2; + } + + Directory.CreateDirectory(outputFull); + foreach (string name in new[] { "coverage.csv", "rules.csv" }) + { + File.Copy(Path.Combine(scratch, name), Path.Combine(outputFull, name), overwrite: true); + Console.WriteLine($"wrote {Path.Combine(outputFull, name)}"); + } + } + } + finally + { + Directory.Delete(scratch, recursive: true); + } + + // Absolute construct-coverage check against constructs.txt: every construct except Tracing + // must be covered. + if (File.Exists(constructsPath)) + { + List checklist = CoverageReport.LoadConstructChecklist(constructsPath); + List uncovered = checklist + .Where(c => !string.Equals(c, OutOfScopeConstruct, StringComparison.Ordinal)) + .Where(c => !result.CoveredConstructs.Contains(c)) + .ToList(); + Console.WriteLine(); + int inScope = checklist.Count(c => !string.Equals(c, OutOfScopeConstruct, StringComparison.Ordinal)); + Console.WriteLine( + $"construct coverage: {inScope - uncovered.Count}/{inScope} in-scope constructs covered " + + $"(Tracing out of scope by design)" + ); + if (uncovered.Count > 0) + { + failed = true; + Console.WriteLine($"*** {uncovered.Count} CONSTRUCT(S) AT ZERO COVERAGE ***"); + foreach (string c in uncovered) + Console.WriteLine($" {c}"); + } + } + else + { + failed = true; + Console.WriteLine(); + Console.WriteLine($"*** CONSTRUCT CHECKLIST NOT FOUND: {constructsPath} ***"); + } + + if (result.UnknownBlockedBy.Count > 0) + { + failed = true; + Console.WriteLine(); + Console.WriteLine( + $"*** {result.UnknownBlockedBy.Count} BLOCKED_BY LABEL(S) ARE NOT RULE IDS IN THEIR FIXTURE'S GRAMMAR ***" + ); + foreach (CoverageReport.UnknownBlockedBy unknown in result.UnknownBlockedBy) + Console.WriteLine($" {unknown.FixtureId}: word '{unknown.Word}' blocked_by '{unknown.Label}'"); + } + + string allowlistPath = Path.Combine(fixturesRoot, "dead-rules-allowed.tsv"); + Dictionary<(string FixtureId, string RuleId), string> allowed = File.Exists(allowlistPath) + ? CoverageReport.LoadDeadRuleAllowlist(allowlistPath) + : new Dictionary<(string FixtureId, string RuleId), string>(); + (List unexpected, List<(string FixtureId, string RuleId)> stale) = + CoverageReport.CheckDeadRules(result.DeadRules, allowed); + + Console.WriteLine(); + Console.WriteLine( + $"dead rules: {result.DeadRules.Count} (exercised by zero words), " + + $"{result.DeadRules.Count - unexpected.Count} allowed by {Path.GetFileName(allowlistPath)}" + ); + if (unexpected.Count > 0) + { + failed = true; + Console.WriteLine($"*** {unexpected.Count} DEAD RULE(S) NOT IN {Path.GetFileName(allowlistPath)} ***"); + foreach (CoverageReport.DeadRule dead in unexpected) + Console.WriteLine($" {dead.FixtureId}: rule '{dead.RuleId}'"); + } + if (stale.Count > 0) + { + failed = true; + Console.WriteLine($"*** {stale.Count} ALLOWED RULE(S) NOW EXERCISED; REMOVE THEIR ENTRIES ***"); + foreach ((string fixtureId, string ruleId) in stale) + Console.WriteLine($" {fixtureId}: rule '{ruleId}'"); + } + return failed ? 1 : 0; + } + + private static void PrintUsage() + { + Console.WriteLine( + """ + usage: hc-conformance --fixtures [options] + + Runs every fixture under (the conformance/ clone, or conformance/stress) and compares + each word with its words.yaml: against the C# engine in process, or through --adapter. + + options: + --adapter "" Run fixtures through an external engine adapter. + Template placeholders: {grammar} {words} {output} + and {morpher}. Omit to run the harness's self-check mode + (in-process C# engine) instead. + --capabilities Capability set the engine under test declares (e.g. + "phonology", or "" for none). Defaults to the full set + in self-check mode, empty in --adapter mode. A fixture that + requires more is skipped. + --include-pathological Also run fixtures with a budget_ms (excluded by default). + --coverage-report Print a coverage report instead of running fixtures; + exits 1 on a stale coverage.csv or rules.csv, an + uncovered construct or an unallowed dead rule. Writes + nothing under --fixtures. + --coverage-output With --coverage-report, also write the fresh + coverage.csv and rules.csv into , which must be + outside --fixtures (a clone of the suite). + --constructs Construct checklist file for --coverage-report + (default: /constructs.txt). + --check-manifest Check the suite's generated/hc-conformance-manifest.v2.json against + a fresh computation, and every grammar and words.yaml against the + suite's DTD and schema. Writes nothing. + --propose Self-check only: on a signature mismatch, print the words.yaml + patch that would reconcile it. Never writes any file. + """ + ); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d8e979b31 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("SIL.Machine.Morphology.HermitCrab.Tests")] diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs new file mode 100644 index 000000000..3ac446c21 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// --propose: on a self-check signature mismatch, prints the words.yaml patch that +/// would reconcile the fixture with what the oracle actually produced, for a human to accept by +/// hand. This class only ever writes to the +/// it is given -- it has no file-write call anywhere, by construction, so +/// there is no code path here that could accidentally start writing words.yaml back out. +/// +public static class ProposePatchWriter +{ + public static void WriteSignatureProposal( + TextWriter output, + Fixture fixture, + WordEntry word, + List actualEntries, + bool wasSkipped + ) + { + output.WriteLine(); + output.WriteLine($"# --propose: {fixture.Id}/words.yaml -- word '{word.Word}' signature mismatch"); + output.WriteLine("# Proposed replacement entry (rules:/gloss: left as-is; fill in by hand):"); + if (actualEntries.Count == 0) + { + output.WriteLine($" - word: {word.Word}"); + output.WriteLine($" note: {word.Note}"); + // Self-check rejects expect_fail on a word the oracle skipped, so propose what it accepts. + output.WriteLine(wasSkipped ? " expect_skip: true" : " expect_fail: true"); + } + else + { + output.WriteLine($" - word: {word.Word}"); + output.WriteLine($" note: {word.Note}"); + output.WriteLine(" parses:"); + foreach (string entry in actualEntries) + { + output.WriteLine($" - signature: \"{entry}\""); + output.WriteLine(" rules: [] # TODO: fill in from the trace"); + } + } + output.WriteLine(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs new file mode 100644 index 000000000..e999369be --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.Linq; +using System.Xml; +using System.Xml.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public static class RequiresDerivation +{ + /// + /// phonology when the grammar has a phonological rule, plus + /// when any fixture run declares a morpher: block, in that order. + /// + public static List Derive(string grammarPath, MorpherSettings morpher) + { + return Derive(grammarPath, hasMorpherSettings: !morpher.IsEmpty); + } + + public static List Derive(string grammarPath, IEnumerable runs) + { + return Derive(grammarPath, hasMorpherSettings: runs.Any(settings => !settings.IsEmpty)); + } + + /// + /// Derive grammar capabilities and add the fixture-level settings capability when any run has + /// settings. A selected run's settings may be empty while another named run requires support. + /// + public static List Derive(string grammarPath, bool hasMorpherSettings) + { + List requires = Derive(grammarPath); + if (hasMorpherSettings) + requires.Add(MorpherSettings.Capability); + return requires; + } + + private static List Derive(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + bool hasPhonology = doc.Descendants() + .Any(e => e.Name.LocalName == "PhonologicalRule" || e.Name.LocalName == "MetathesisRule"); + + return hasPhonology ? new List { "phonology" } : new List(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs new file mode 100644 index 000000000..f3a283e82 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs @@ -0,0 +1,434 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public class RunReport +{ + public List Results { get; } = new(); + public int FixtureCount { get; set; } + public int ExcludedPathologicalCount { get; set; } + + public int Passed => Results.Count(r => r.Outcome == FixtureOutcome.Passed); + public int Failed => Results.Count(r => r.Outcome == FixtureOutcome.Failed); + public int Skipped => Results.Count(r => r.Outcome == FixtureOutcome.Skipped); +} + +/// +/// Runs fixtures under conformance/languages and conformance/edge-cases. Self-check mode is a +/// custom, in-process loop (it needs per-word signature-SET comparison against words.yaml and +/// traced-rules verification, neither of which / +/// does). Adapter mode instead materializes each fixture onto disk and hands it to +/// + -- see . +/// +public static class Runner +{ + /// The capabilities the C# engine declares. A fixture that requires another is skipped. + public static readonly IReadOnlySet DefaultCapabilities = new HashSet + { + "phonology", + MorpherSettings.Capability, + }; + + public static RunReport RunSelfCheck( + List fixtures, + bool includePathological, + IReadOnlySet engineCapabilities, + bool propose, + TextWriter proposeOutput + ) + { + var report = new RunReport(); + report.FixtureCount = fixtures.Count; + foreach (Fixture fixture in fixtures) + { + foreach (WordRun run in fixture.Words.Runs) + { + FixtureResult result = RunOneSelfCheck( + fixture, + run, + includePathological, + engineCapabilities, + propose, + proposeOutput + ); + if (result == null) + report.ExcludedPathologicalCount++; + else + report.Results.Add(result); + } + } + return report; + } + + private static FixtureResult RunOneSelfCheck( + Fixture fixture, + WordRun run, + bool includePathological, + IReadOnlySet engineCapabilities, + bool propose, + TextWriter proposeOutput + ) + { + string runId = run.Name == "default" ? fixture.Id : $"{fixture.Id} [variant {run.Name}]"; + var result = new FixtureResult { FixtureId = runId }; + + List derivedRequires = RequiresDerivation.Derive( + fixture.GrammarPath, + fixture.Words.Runs.Select(r => r.Morpher) + ); + if ( + !new HashSet(derivedRequires, StringComparer.Ordinal).SetEquals( + new HashSet(fixture.Words.Requires, StringComparer.Ordinal) + ) + ) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"words.yaml error: requires [{string.Join(",", fixture.Words.Requires)}] does not match " + + $"grammar-derived [{string.Join(",", derivedRequires)}]"; + return result; + } + + bool isPathological = fixture.Words.BudgetMs.HasValue; + if (isPathological && !includePathological) + return null; + + List missingCapabilities = fixture.Words.Requires.Where(r => !engineCapabilities.Contains(r)).ToList(); + if (missingCapabilities.Count > 0) + { + result.Outcome = FixtureOutcome.Skipped; + result.Reason = + $"requires [{string.Join(",", fixture.Words.Requires)}], missing [{string.Join(",", missingCapabilities)}]"; + return result; + } + + var sw = Stopwatch.StartNew(); + try + { + RunWithBudget(fixture, run, result, propose, proposeOutput); + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Words.ExpectCrash) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = "expected crash, engine produced output"; + } + else + { + bool overBudget = fixture.Words.BudgetMs.HasValue && result.ElapsedMs > fixture.Words.BudgetMs.Value; + if (overBudget) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"exceeded wall-clock budget ({result.ElapsedMs}ms > {fixture.Words.BudgetMs}ms)"; + } + else if (result.WordResults.All(w => w.Passed)) + { + result.Outcome = FixtureOutcome.Passed; + } + else + { + result.Outcome = FixtureOutcome.Failed; + // Name the words. A bare count cannot be diagnosed from a CI log on a machine you cannot reach. + WordResult[] mismatched = result.WordResults.Where(word => !word.Passed).ToArray(); + int total = result.WordResults.Count; + string named = string.Join( + "; ", + mismatched.Take(5).Select(word => $"{word.Word}: {Truncate(word.Detail)}") + ); + string more = mismatched.Length > 5 ? $"; and {mismatched.Length - 5} more" : ""; + result.Reason = $"{mismatched.Length}/{total} word(s) mismatched: {named}{more}"; + } + } + } + catch (Exception ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + // The watchdog's own timeout is never a crash match, as in adapter mode. + if (fixture.Words.ExpectCrash && ex is not TimeoutException) + { + result.Outcome = FixtureOutcome.Passed; + result.Reason = $"engine crashed as expected (words.yaml expect_crash: true): {ex.Message}"; + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"engine crashed in run '{run.Name}': {ex.Message}"; + } + } + + return result; + } + + // Mirrors MaterializedRunner.RunWithBudget's watchdog shape (see that method's doc comment) -- a + // budgeted fixture gets the same "background thread + Task.Wait" timeout enforcement. + private static void RunWithBudget( + Fixture fixture, + WordRun run, + FixtureResult result, + bool propose, + TextWriter proposeOutput + ) + { + if (!fixture.Words.BudgetMs.HasValue) + { + RunAllWords(fixture, run, result.WordResults, propose, proposeOutput, CancellationToken.None); + return; + } + + long budgetMs = fixture.Words.BudgetMs.Value; + // The task fills its own list, so a task abandoned on timeout cannot mutate the reported result. + var wordResults = new List(); + using var cancellation = new CancellationTokenSource(); + Task task = Task.Run(() => RunAllWords(fixture, run, wordResults, propose, proposeOutput, cancellation.Token)); + long marginMs = Math.Max(budgetMs / 5, 1000); + bool completed; + try + { + completed = task.Wait((int)(budgetMs + marginMs)); + } + catch (AggregateException ex) when (ex.InnerException != null) + { + throw ex.InnerException; + } + if (!completed) + { + cancellation.Cancel(); + throw new TimeoutException("watchdog timed out waiting for self-check engine"); + } + result.WordResults.AddRange(wordResults); + } + + private static void RunAllWords( + Fixture fixture, + WordRun run, + List wordResults, + bool propose, + TextWriter proposeOutput, + CancellationToken cancellationToken + ) + { + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + Morpher morpher = ConformanceMorpherFactory.CreateTracing(language, run.Morpher); + GrammarRuleIndex ruleIndex = GrammarRuleIndex.Load(fixture.GrammarPath); + + foreach (WordEntry word in run.Words) + { + cancellationToken.ThrowIfCancellationRequested(); + bool guessRoot = run.Morpher.GuessRootsFor(word); + List results; + object trace = null; + bool wasSkipped = false; + try + { + results = morpher.ParseWord(word.Word, out trace, guessRoot).ToList(); + } + catch (InvalidShapeException) + { + wasSkipped = true; + results = new List(); + } + + string actualSignature = SignatureFormat.BuildSignature(results); + string expectedSignature = FixtureMaterializer.BuildExpectedSignature(word); + List actualEntries = SignatureTsv.SplitSignature(actualSignature); + List expectedEntries = SignatureTsv.SplitSignature(expectedSignature); + + bool signatureMatches = SignatureTsv + .SplitComparableSignature(actualSignature) + .SequenceEqual(SignatureTsv.SplitComparableSignature(expectedSignature), StringComparer.Ordinal); + if (!signatureMatches) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = false, + Detail = + $"expected [{string.Join(" | ", expectedEntries)}] got [{string.Join(" | ", actualEntries)}]", + } + ); + if (propose && run.Name == "default") + { + ProposePatchWriter.WriteSignatureProposal(proposeOutput, fixture, word, actualEntries, wasSkipped); + } + else if (propose) + { + proposeOutput.WriteLine( + $"# --propose cannot safely patch variant '{run.Name}' in {fixture.Id}; edit that variant's complete words list by hand." + ); + } + continue; + } + + if (word.ExpectSkip) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = wasSkipped, + Detail = wasSkipped + ? "ok (skipped: InvalidShapeException)" + : "declared expect_skip but engine produced a zero-parse 'ok' without skipping", + } + ); + continue; + } + + if (word.ExpectFail) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = !wasSkipped, + Detail = !wasSkipped + ? "ok (0 parses)" + : "declared expect_fail (status ok) but engine SKIPPED it (InvalidShapeException) -- use expect_skip", + } + ); + continue; + } + + HashSet wordLevelRuleIds = TraceRuleAttributor.WordLevelRuleIds(trace, ruleIndex); + + List ruleDetails = CompareRuleTraceMultisets( + word.Parses.Select(parse => (parse.Signature, (IEnumerable)parse.Rules)), + results.Select(match => + { + string signature = SignatureFormat.BuildSignature(new[] { match }); + HashSet actualRuleIds = TraceRuleAttributor.MorphologicalRuleIds(match, ruleIndex); + actualRuleIds.UnionWith(wordLevelRuleIds); + return (signature, (IEnumerable)actualRuleIds); + }) + ); + bool wordRulesOk = ruleDetails.Count == 0; + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = wordRulesOk, + Detail = wordRulesOk ? "ok" : string.Join("; ", ruleDetails), + } + ); + } + } + + internal static List CompareRuleTraceMultisets( + IEnumerable<(string Signature, IEnumerable RuleIds)> expected, + IEnumerable<(string Signature, IEnumerable RuleIds)> actual + ) + { + static string CanonicalRuleIds(IEnumerable ruleIds) => + string.Join( + '\u001f', + ruleIds.Distinct(StringComparer.Ordinal).OrderBy(ruleId => ruleId, StringComparer.Ordinal) + ); + + static Dictionary<(string Signature, string RuleIds), int> CountTraces( + IEnumerable<(string Signature, IEnumerable RuleIds)> traces + ) + { + var counts = new Dictionary<(string Signature, string RuleIds), int>(); + foreach ((string signature, IEnumerable ruleIds) in traces) + { + var key = (SignatureTsv.CanonicalizeBracketClasses(signature), CanonicalRuleIds(ruleIds)); + counts[key] = counts.GetValueOrDefault(key) + 1; + } + return counts; + } + + Dictionary<(string Signature, string RuleIds), int> expectedCounts = CountTraces(expected); + Dictionary<(string Signature, string RuleIds), int> actualCounts = CountTraces(actual); + var details = new List(); + foreach ( + (string Signature, string RuleIds) key in expectedCounts + .Keys.Union(actualCounts.Keys) + .OrderBy(key => key.Signature, StringComparer.Ordinal) + .ThenBy(key => key.RuleIds, StringComparer.Ordinal) + ) + { + int declaredCount = expectedCounts.GetValueOrDefault(key); + int tracedCount = actualCounts.GetValueOrDefault(key); + if (declaredCount == tracedCount) + continue; + + string declaredRuleIds = string.Join( + ',', + key.RuleIds.Length == 0 ? Array.Empty() : key.RuleIds.Split('\u001f') + ); + details.Add( + $"parse '{key.Signature}': rules [{declaredRuleIds}]: declared {declaredCount}, " + + $"traced {tracedCount}" + ); + } + + return details; + } + + // ---- Adapter mode: materialize onto disk + reuse MaterializedRunner/AdapterEngine, unmodified. ---- + + public static RunReport RunAdapter(List fixtures, IEngine engine, bool includePathological) + { + var report = new RunReport(); + report.FixtureCount = fixtures.Count; + // One call over every run of every fixture, so an expect_crash fixture has an ordinary fixture to use + // as its control. + var materialized = new List(); + try + { + foreach (Fixture fixture in fixtures) + { + foreach (WordRun run in fixture.Words.Runs) + { + try + { + materialized.Add(FixtureMaterializer.Materialize(fixture, run)); + } + catch (NamedVariantHasNoAdapterVisibleWordsException ex) + { + report.Results.Add( + new FixtureResult + { + FixtureId = run.Name == "default" ? fixture.Id : $"{fixture.Id} [variant {run.Name}]", + Outcome = FixtureOutcome.Failed, + Reason = ex.Message, + } + ); + } + } + } + + MaterializedRunReport runReport = MaterializedRunner.Run(materialized, engine, includePathological); + report.ExcludedPathologicalCount = runReport.ExcludedPathologicalCount; + report.Results.AddRange(runReport.Results); + } + finally + { + foreach (MaterializedFixture fixture in materialized) + FixtureMaterializer.Cleanup(fixture); + } + return report; + } + + /// Shortens a per-word detail so a fixture reason stays readable in a log. + /// The word detail, which may carry every expected and actual parse. + private static string Truncate(string detail) + { + const int Limit = 160; + if (string.IsNullOrEmpty(detail) || detail.Length <= Limit) + { + return detail; + } + + return detail[..Limit] + "..."; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj new file mode 100644 index 000000000..3d8818af0 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj @@ -0,0 +1,32 @@ + + + + Exe + net10.0 + SIL.Machine.Morphology.HermitCrab.Conformance + hc-conformance + Runs the HermitCrab conformance suite (the clone in conformance/) against the C# engine. + SIL.Machine.Morphology.HermitCrab.Conformance + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + + + diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs new file mode 100644 index 000000000..d900006c0 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Runs a fixture against the live C# HermitCrab engine, in-process: loads grammar.xml via +/// , builds a , parses every word in +/// words.txt, and computes the exact same signature that +/// BatchCommand's oracle uses -- byte-identical, since it's the same shared code, not a +/// reimplementation. This is what catches a fixture going stale when the oracle itself changes. +/// +/// +/// Lets a caller override the reference engine's normal full-capability declaration -- e.g. +/// --capabilities "", which is how the harness's own capability-filtering skip path gets +/// exercised against the live oracle without needing a second, capability-limited engine on hand. +/// Null (the default) means the full set. +/// +public class SelfCheckEngine(IReadOnlySet capabilities = null) : IEngine +{ + public string Name => "self-check"; + + // The reference C# engine implements every capability the suite currently knows about, unless + // overridden above. + public IReadOnlySet Capabilities { get; } = capabilities ?? Runner.DefaultCapabilities; + + public List Run(MaterializedFixture fixture) + { + Language language; + Morpher morpher; + try + { + language = XmlLanguageLoader.Load(fixture.GrammarPath); + morpher = ConformanceMorpherFactory.Create(language, fixture.Manifest.Morpher); + } + catch (Exception ex) + { + throw new EngineCrashException( + $"self-check engine crashed loading grammar for fixture '{fixture.Id}': {ex.Message}", + ex + ); + } + + string[] words = SignatureFormat.LoadWords(fixture.WordsPath); + + var rows = new List(words.Length); + for (int i = 0; i < words.Length; i++) + { + string word = words[i]; + try + { + (string status, _, string signature) = SignatureFormat.ParseOneWord( + morpher, + word, + fixture.Manifest.Morpher.GuessRoots ?? false + ); + rows.Add(new TsvRow(word, status, signature)); + } + catch (Exception ex) + { + throw new EngineCrashException( + $"self-check engine crashed parsing word '{word}' in fixture '{fixture.Id}': {ex.Message}", + ex + ); + } + } + return rows; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs new file mode 100644 index 000000000..3aaf3106d --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public readonly struct TsvRow(string word, string status, string signature) +{ + public readonly string Word = word; + public readonly string Status = status; + public readonly string Signature = signature; +} + +public static class SignatureTsv +{ + /// Reads a batch TSV file, silently skipping malformed/sentinel lines. + public static List ReadFile(string path) + { + var rows = new List(); + foreach (string line in File.ReadAllLines(path)) + { + if (TryParseRow(line, out TsvRow row)) + rows.Add(row); + } + return rows; + } + + public static bool TryParseRow(string line, out TsvRow row) + { + row = default; + if (string.IsNullOrEmpty(line)) + return false; + string[] cols = line.Split('\t'); + // A well-formed row has exactly 5 columns (idx, word, ms, status, signature). The + // 3-column "STARTED" sentinel and any other malformed line is not comparison data. + if (cols.Length != 5) + return false; + if (!int.TryParse(cols[0], out _)) + return false; + if (!long.TryParse(cols[2], out _)) + return false; + + if (cols[3] != "ok" && cols[3] != "SKIPPED") + return false; + row = new TsvRow(cols[1], cols[3], cols[4]); + return true; + } + + /// The ";"-separated signature entries as a multiset (sorted list, for equality/diff display). + public static List SplitSignature(string signature) + { + if (signature == "-") + return new List(); + return signature.Split(';').OrderBy(s => s, StringComparer.Ordinal).ToList(); + } + + private static readonly Regex BracketClass = new(@"\[(\^?)([^\[\]]*)\]", RegexOptions.Compiled); + + /// + /// with the members of every bracketed class sorted, so "[ea]" equals "[ae]". + /// HermitCrab renders class members in character-table insertion order, which FieldWorks fills from an + /// unordered collection (sillsdev/machine#573). + /// + public static List SplitComparableSignature(string signature) => + SplitSignature(signature).Select(CanonicalizeBracketClasses).OrderBy(s => s, StringComparer.Ordinal).ToList(); + + /// One signature entry with the members of every bracketed class sorted. + public static string CanonicalizeBracketClasses(string entry) => + BracketClass.Replace( + entry, + m => "[" + m.Groups[1].Value + new string(m.Groups[2].Value.OrderBy(c => c).ToArray()) + "]" + ); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs new file mode 100644 index 000000000..d4982f6ff --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Warns when a run reads the repository's conformance/ clone while it is not at the commit in +/// conformance.commit, which dotnet run --no-build would otherwise do silently after a pin edit, +/// or while it holds local changes. A --fixtures path outside such a clone, or +/// HC_CONFORMANCE_SUITE being set, means the caller chose the suite and nothing is checked. +/// +internal static class SuitePinCheck +{ + private const string OverrideVariable = "HC_CONFORMANCE_SUITE"; + + public static IReadOnlyList Warnings(string fixturesRoot) + { + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(OverrideVariable))) + return []; + + string pinFile = null; + string suite = null; + for ( + DirectoryInfo directory = new DirectoryInfo(Path.GetFullPath(fixturesRoot)); + directory?.Parent != null; + directory = directory.Parent + ) + { + string candidate = Path.Combine(directory.Parent.FullName, "conformance.commit"); + if (directory.Name == "conformance" && File.Exists(candidate)) + { + pinFile = candidate; + suite = directory.FullName; + break; + } + } + if (suite == null || !Directory.Exists(Path.Combine(suite, ".git"))) + return []; + + string pin = File.ReadAllText(pinFile).Trim(); + string head = Git(suite, "rev-parse", "HEAD")?.Trim(); + if (head == null) + return [$"could not run git in {suite}, so it was not checked against conformance.commit ({pin})"]; + + var warnings = new List(); + if (head != pin) + { + warnings.Add( + $"{suite} is at {head}, not the commit in conformance.commit ({pin}), so this run reads a stale suite. " + + "Build the runner (dotnet build src/SIL.Machine.Morphology.HermitCrab.Conformance) to move it." + ); + } + + string status = Git(suite, "status", "--porcelain"); + string[] changed = (status ?? "") + .Split('\n') + .Select(line => line.TrimEnd('\r')) + .Where(line => line.Length > 3) + .Select(line => line.Substring(3)) + .ToArray(); + if (changed.Length > 0) + { + warnings.Add( + $"{suite} has {changed.Length} local change(s): {string.Join(", ", changed.Take(5))}" + + (changed.Length > 5 ? ", ..." : "") + + ". This run uses what is on disk." + ); + } + return warnings; + } + + private static string Git(string workingDirectory, params string[] arguments) + { + var startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = workingDirectory; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.EnvironmentVariables["GIT_OPTIONAL_LOCKS"] = "0"; + foreach (string variable in new[] { "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE" }) + startInfo.EnvironmentVariables.Remove(variable); + foreach (string argument in arguments) + startInfo.ArgumentList.Add(argument); + try + { + using var process = System.Diagnostics.Process.Start(startInfo); + string output = process.StandardOutput.ReadToEnd(); + process.StandardError.ReadToEnd(); + process.WaitForExit(); + return process.ExitCode == 0 ? output : null; + } + catch (System.ComponentModel.Win32Exception) + { + return null; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs new file mode 100644 index 000000000..e05a250d6 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Extracts, for one successful parse () returned by Morpher.ParseWord, +/// the set of grammar rule ids a word's "rules:" field is verified against: self-check runs the +/// oracle with tracing enabled and fails if the traced rule applications don't match the declared +/// list. +/// +/// Two different sources, because no single API gives both: +/// +/// Morphological rules (AffixProcessRule/CompoundingRule) come straight off +/// -- populated once during analysis-direction unapplication +/// and carried forward unchanged through synthesis, so it is exact and per-result, no ambiguity. +/// Phonological rules (RewriteRule/MetathesisRule) have no equivalent per-Word list -- +/// only records their application as nodes +/// (), via . +/// Unlike , that call does NOT update +/// Word.CurrentTrace (see TraceManager.cs), so a phonological-rule trace node is a *sibling* +/// of the branch that produced a given final Word, not an ancestor reachable by walking +/// CurrentTrace upward. This class instead walks the ENTIRE trace tree for the word (the +/// trace out-parameter of ParseWord) and collects every +/// node system-wide. +/// +/// +/// Documented limitation (ambiguous words): when a surface word has more than one successful +/// analysis, this global trace walk cannot attribute a specific phonological rule firing to one +/// analysis branch over another -- it reports the UNION of every phonological rule that fired +/// anywhere while producing any of the word's results. applies this same +/// phonological-rule set to every one of that word's declared parses, so a false PASS is possible +/// in principle for an ambiguous word whose two parses use different phonological rules but the +/// same declared "rules:" set does not distinguish them. This does not affect any G1 fixture (the +/// pilot grammar has zero phonological rules by construction; the one ambiguous pilot word's +/// parses differ only in a morphological rule, which IS attributed exactly per-result). A future +/// language with both ambiguity and phonological rules would need per-branch trace correlation +/// (e.g. threading a distinguishing marker through ITraceManager.PhonologicalRuleApplied) to +/// close this gap -- flagged here rather than silently shipped. +/// +/// RealizationalAffixProcessRule applications are attributed word-level too: +/// Word.MorphologicalRuleUnapplied deliberately excludes them from the _mruleApps +/// list that backs (see Word.cs), so they cannot come from +/// the per-result source above -- but SynthesisRealizationalAffixProcessRule does report +/// successful applications through , which +/// records a node. The same trace walk that +/// collects phonological rules therefore also collects realizational rules (with the same +/// union-across-results ambiguity caveat), keyed off the node's Source type. +/// +public static class TraceRuleAttributor +{ + public static HashSet MorphologicalRuleIds(Word result, GrammarRuleIndex index) + { + var ids = new HashSet(StringComparer.Ordinal); + foreach (IMorphologicalRule rule in result.MorphologicalRules) + { + string id = index.ResolveMorphologicalRuleId(rule); + if (id != null) + ids.Add(id); + } + return ids; + } + + /// The word-level (not per-result) set of phonological and realizational rule ids fired + /// anywhere in the trace tree rooted at -- see the class doc comment for + /// why these two kinds are word-level, not per-result. + public static HashSet WordLevelRuleIds(object trace, GrammarRuleIndex index) + { + var ids = new HashSet(StringComparer.Ordinal); + if (trace is not Trace root) + return ids; + Walk(root); + return ids; + + void Walk(Trace node) + { + if (node.FailureReason == FailureReason.None) + { + if (node.Type == TraceType.PhonologicalRuleSynthesis && node.Source is IPhonologicalRule prule) + { + string id = index.ResolvePhonologicalRuleId(prule); + if (id != null) + ids.Add(id); + } + else if ( + node.Type == TraceType.MorphologicalRuleSynthesis + && node.Source is RealizationalAffixProcessRule rrule + ) + { + string id = index.ResolveMorphologicalRuleId(rrule); + if (id != null) + ids.Add(id); + } + } + foreach (Trace child in node.Children) + Walk(child); + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs new file mode 100644 index 000000000..ac93197ce --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs @@ -0,0 +1,142 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json.Nodes; +using Json.Schema; +using YamlDotNet.RepresentationModel; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Validates an authored words.yaml against the published words schema. +/// The schema, not the loader, is what a consumer outside this repository can read, so the +/// two must agree on every checked-in fixture. Plain YAML only: aliases, anchors, merge keys and +/// custom tags are refused before conversion rather than silently resolved. +public static class WordsSchemaValidation +{ + public const string SchemaRelativePath = "schema/words.schema.json"; + + // A schema registers itself globally under its $id, so it is loaded once per path rather than + // once per file validated. + private static readonly Dictionary LoadedSchemas = new(StringComparer.OrdinalIgnoreCase); + + /// Validation messages for ; empty when it conforms. + public static IReadOnlyList Validate(string wordsPath, string schemaPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(wordsPath); + ArgumentException.ThrowIfNullOrWhiteSpace(schemaPath); + + JsonNode? node = ReadAsJson(wordsPath); + if (node is null) + return new[] { "words.yaml is empty" }; + + using var document = System.Text.Json.JsonDocument.Parse(node.ToJsonString()); + EvaluationResults results = Schema(schemaPath) + .Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); + if (results.IsValid) + return Array.Empty(); + + return (results.Details ?? new List()) + .Where(detail => detail.Errors is { Count: > 0 }) + .SelectMany(detail => detail.Errors!.Select(error => $"{detail.InstanceLocation}: {error.Value}")) + .DefaultIfEmpty("does not conform to " + SchemaRelativePath) + .ToArray(); + } + + private static JsonSchema Schema(string schemaPath) + { + string key = Path.GetFullPath(schemaPath); + lock (LoadedSchemas) + { + if (!LoadedSchemas.TryGetValue(key, out JsonSchema? schema)) + { + schema = JsonSchema.FromFile(key); + LoadedSchemas[key] = schema; + } + + return schema; + } + } + + private static JsonNode? ReadAsJson(string wordsPath) + { + var stream = new YamlStream(); + using (var reader = new StreamReader(wordsPath)) + stream.Load(reader); + if (stream.Documents.Count == 0) + return null; + if (stream.Documents.Count != 1) + throw new InvalidDataException($"{wordsPath}: a words file must contain exactly one YAML document."); + + return Convert(stream.Documents[0].RootNode, wordsPath); + } + + private static JsonNode? Convert(YamlNode node, string path) + { + if (!node.Anchor.IsEmpty) + throw new InvalidDataException($"{path}: YAML anchors are not admitted."); + + switch (node) + { + case YamlMappingNode mapping: + { + var result = new JsonObject(); + foreach (KeyValuePair pair in mapping.Children) + { + if (pair.Key is not YamlScalarNode { Value: { } key }) + throw new InvalidDataException($"{path}: mapping keys must be scalars."); + if (key == "<<") + throw new InvalidDataException($"{path}: YAML merge keys are not admitted."); + result[key] = Convert(pair.Value, path); + } + + return result; + } + + case YamlSequenceNode sequence: + { + var result = new JsonArray(); + foreach (YamlNode child in sequence.Children) + result.Add(Convert(child, path)); + return result; + } + + case YamlScalarNode scalar: + return ConvertScalar(scalar); + + default: + throw new InvalidDataException($"{path}: unsupported YAML node '{node.GetType().Name}'."); + } + } + + /// Applies YAML 1.1 core scalar resolution for the types the schema distinguishes. + private static JsonNode? ConvertScalar(YamlScalarNode scalar) + { + string? value = scalar.Value; + if (value is null) + return null; + if ( + scalar.Style + is YamlDotNet.Core.ScalarStyle.SingleQuoted + or YamlDotNet.Core.ScalarStyle.DoubleQuoted + or YamlDotNet.Core.ScalarStyle.Literal + or YamlDotNet.Core.ScalarStyle.Folded + ) + { + return JsonValue.Create(value); + } + + if (value is "null" or "~" or "") + return null; + if (value is "true" or "True" or "TRUE") + return JsonValue.Create(true); + if (value is "false" or "False" or "FALSE") + return JsonValue.Create(false); + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out long integer)) + return JsonValue.Create(integer); + + return JsonValue.Create(value); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs new file mode 100644 index 000000000..eb95d20a7 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs @@ -0,0 +1,94 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// In-memory model of one words.yaml file. Parsed by , which +/// enforces the strict, fixed key vocabulary -- this class only carries already-validated data, it +/// does not itself validate anything. +/// +public class WordsYaml +{ + public string Language = ""; + public List InspiredBy { get; } = new(); + public List Sources { get; } = new(); + public List Requires { get; } = new(); + + /// Edge-cases only: a wall-clock budget in milliseconds for the whole fixture. + public long? BudgetMs; + + /// Edge-cases only: this fixture's ground truth is a crash, not a signature. + public bool ExpectCrash; + + public bool? FieldworksProducible; + + /// Required and non-empty when is false: names the + /// offending construct(s) and cites the HCLoader evidence grounding the verdict. + public string FieldworksProducibleNotes = ""; + + /// The morpher: block; empty when absent, meaning the engine constructor defaults. + public MorpherSettings Morpher { get; } = new(); + + public List Words { get; } = new(); + + /// + /// Named complete runs against the same grammar; coverage remains attached to . + /// + public List Variants { get; } = new(); + + public IEnumerable Runs + { + get + { + yield return new WordRun("default", Morpher, Words); + foreach (WordRun variant in Variants) + yield return variant; + } + } +} + +public class WordRun(string name, MorpherSettings morpher, List words) +{ + public string Name { get; } = name; + public MorpherSettings Morpher { get; } = morpher; + public List Words { get; } = words; +} + +public class WordEntry +{ + public string Word = ""; + public string Note = ""; + public string Provenance = ""; + public bool ExpectFail; + + /// + /// The oracle SKIPS this word rather than returning a genuine zero-parse "ok": it throws + /// (e.g. the surface contains a segment undeclared in the + /// grammar's CharacterDefinitionTable). Distinct from (well-formed input, + /// zero valid analyses) because the two produce different status columns -- + /// "SKIPPED" vs "ok" -- in BatchCommand's output. The runner confirms an + /// was actually thrown. Like expect_fail, a skip word has no parses. + /// + public bool ExpectSkip; + + public List BlockedBy { get; } = new(); + + /// Declaration ids (a grammar element's id) whose deactivation this word's failure observes. + /// The runner does not read it; the loader accepts it because the suite's schema allows it. + public List Neutralizes { get; } = new(); + public List Exercises { get; } = new(); + public List Parses { get; } = new(); + + /// The cell of each claims entry. The loader checks the entry's shape, since the suite's + /// schema allows it; the runner does not otherwise read it. + public List Claims { get; } = new(); +} + +public class ParseEntry +{ + public string Signature = ""; + public string Gloss = ""; + public bool Guess; + public List Rules { get; } = new(); + public List Exercises { get; } = new(); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs new file mode 100644 index 000000000..ae3aea6bb --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs @@ -0,0 +1,512 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.RepresentationModel; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Strict parser for a fixture's words.yaml: enforces a fixed key vocabulary (unknown keys +/// are hard errors) and a plain YAML 1.2 subset only -- no anchors, aliases, merge keys, or custom +/// tags. +/// +public static class WordsYamlLoader +{ + // The complete, fixed key vocabulary at each of the three mapping levels. Anything else + // encountered is a hard parse error -- this is the mechanism behind "unknown keys are errors". + private static readonly HashSet FrontMatterKeys = new(StringComparer.Ordinal) + { + "language", + "inspired_by", + "sources", + "requires", + "budget_ms", + "expect_crash", + "fieldworks_producible", + "fieldworks_producible_notes", + "morpher", + "words", + "variants", + }; + + // The morpher: block's keys are FieldWorks element names (HCParser.LoadParser), + // limited to the settings whose only effect is which analyses a parse returns. + private static readonly HashSet MorpherKeys = new(StringComparer.Ordinal) + { + "DelReapps", + "MaxRoots", + "MergeAnalyses", + "MakePartialsFinal", + "GuessRoots", + }; + private static readonly HashSet WordKeys = new(StringComparer.Ordinal) + { + "word", + "note", + "provenance", + "expect_fail", + "expect_skip", + "blocked_by", + "neutralizes", + "exercises", + "parses", + "claims", + }; + private static readonly HashSet ParseKeys = new(StringComparer.Ordinal) + { + "signature", + "gloss", + "guess", + "rules", + "exercises", + }; + private static readonly HashSet ClaimKeys = new(StringComparer.Ordinal) + { + "cell", + "severing", + "before", + "after", + "distinct_from", + "proof", + }; + + public static WordsYaml Load(string path) + { + string text = File.ReadAllText(path); + try + { + RejectNonPlainConstructs(text, path); + + var yamlStream = new YamlStream(); + using (var reader = new StringReader(text)) + yamlStream.Load(reader); + + if (yamlStream.Documents.Count != 1) + throw new WordsYamlException(path, "expected exactly one YAML document"); + + if (yamlStream.Documents[0].RootNode is not YamlMappingNode root) + throw new WordsYamlException(path, "top-level YAML node must be a mapping"); + + return ParseFrontMatter(root, path); + } + catch (YamlException ex) + { + throw new WordsYamlException(path, $"YAML syntax error at {ex.Start}-{ex.End}: {ex.Message}", ex); + } + } + + /// + /// Scans the raw parser event stream (not the composed node tree -- by the time + /// composes a tree, an alias has already been resolved into a shared + /// node reference and is no longer visible as such) for anchors, aliases, and non-core tags, + /// which words.yaml forbids along with merge keys and custom tags. Merge keys ("<<") + /// are a mapping-key convention layered on plain scalars, not a distinct event type, so those + /// are rejected in instead, where key strings are actually read. + /// + private static void RejectNonPlainConstructs(string text, string path) + { + var parser = new Parser(new StringReader(text)); + while (parser.MoveNext()) + { + ParsingEvent evt = parser.Current; + if (evt is AnchorAlias) + throw new WordsYamlException(path, "YAML aliases (*name) are not allowed in words.yaml"); + + if (evt is NodeEvent nodeEvent) + { + if (!nodeEvent.Anchor.IsEmpty) + throw new WordsYamlException(path, "YAML anchors (&name) are not allowed in words.yaml"); + if ( + !nodeEvent.Tag.IsEmpty + && !nodeEvent.Tag.Value.StartsWith("tag:yaml.org,2002:", StringComparison.Ordinal) + ) + { + throw new WordsYamlException( + path, + $"custom YAML tag '{nodeEvent.Tag.Value}' is not allowed in words.yaml" + ); + } + } + } + } + + private static WordsYaml ParseFrontMatter(YamlMappingNode root, string path) + { + var doc = new WordsYaml(); + Dictionary map = RequireMapping(root, FrontMatterKeys, path, "front matter"); + + doc.Language = RequireScalar(map, "language", path, "front matter"); + AppendScalarList(map, "inspired_by", doc.InspiredBy, path, "front matter"); + AppendScalarList(map, "sources", doc.Sources, path, "front matter"); + AppendScalarList(map, "requires", doc.Requires, path, "front matter"); + if (map.TryGetValue("budget_ms", out YamlNode budgetNode)) + { + string budget = RequireScalarNode(budgetNode, path, "front matter.budget_ms"); + if (!long.TryParse(budget, NumberStyles.None, CultureInfo.InvariantCulture, out long budgetMs)) + { + throw new WordsYamlException( + path, + $"front matter.budget_ms: expected a whole number of milliseconds, got '{budget}'" + ); + } + doc.BudgetMs = budgetMs; + } + if (map.TryGetValue("expect_crash", out YamlNode crashNode)) + doc.ExpectCrash = ParseBool(RequireScalarNode(crashNode, path, "front matter.expect_crash"), path); + if (map.TryGetValue("fieldworks_producible", out YamlNode fwProducibleNode)) + { + doc.FieldworksProducible = ParseBool( + RequireScalarNode(fwProducibleNode, path, "front matter.fieldworks_producible"), + path + ); + } + if (map.TryGetValue("fieldworks_producible_notes", out YamlNode fwNotesNode)) + { + doc.FieldworksProducibleNotes = RequireScalarNode( + fwNotesNode, + path, + "front matter.fieldworks_producible_notes" + ); + } + if (doc.FieldworksProducible == false && doc.FieldworksProducibleNotes.Length == 0) + { + throw new WordsYamlException( + path, + "front matter: 'fieldworks_producible: false' requires a non-empty 'fieldworks_producible_notes'" + ); + } + + if (map.TryGetValue("morpher", out YamlNode morpherNode)) + ParseMorpher(morpherNode, doc.Morpher, path); + + if (!map.TryGetValue("words", out YamlNode wordsNode) || wordsNode is not YamlSequenceNode wordsSeq) + throw new WordsYamlException(path, "front matter must have a 'words' sequence"); + if (wordsSeq.Children.Count == 0) + throw new WordsYamlException(path, "'words' must contain at least one word entry"); + + foreach (YamlNode wordNode in wordsSeq.Children) + doc.Words.Add(ParseWord(wordNode, path)); + + if (map.TryGetValue("variants", out YamlNode variantsNode)) + ParseVariants(variantsNode, doc, path); + + bool requiresSettings = !doc.Morpher.IsEmpty || doc.Variants.Count > 0; + if (doc.Requires.Contains(MorpherSettings.Capability) != requiresSettings) + { + throw new WordsYamlException( + path, + $"front matter: requires must {(requiresSettings ? "contain" : "not contain")} '{MorpherSettings.Capability}' " + + "when a run declares morpher settings" + ); + } + + return doc; + } + + private static void ParseVariants(YamlNode node, WordsYaml doc, string path) + { + if (node is not YamlSequenceNode variants) + throw new WordsYamlException(path, "front matter.variants: expected a sequence"); + if (variants.Children.Count == 0) + throw new WordsYamlException(path, "front matter.variants: expected at least one named run"); + var names = new HashSet(StringComparer.Ordinal) { "default" }; + foreach (YamlNode variantNode in variants.Children) + { + if (variantNode is not YamlMappingNode variantMap) + throw new WordsYamlException(path, "each 'variants' entry must be a mapping"); + Dictionary fields = RequireMapping( + variantMap, + new HashSet(StringComparer.Ordinal) { "name", "morpher", "words" }, + path, + "variant" + ); + string name = RequireScalar(fields, "name", path, "variant"); + if (name.Any(c => !(c is >= 'a' and <= 'z' or >= '0' and <= '9' or '-'))) + throw new WordsYamlException(path, $"variant name '{name}' must match [a-z0-9-]+"); + if (!names.Add(name)) + throw new WordsYamlException(path, $"duplicate variant name '{name}' (default is reserved)"); + if (!fields.TryGetValue("morpher", out YamlNode morpherNode)) + throw new WordsYamlException(path, $"variant '{name}': missing required key 'morpher'"); + var settings = new MorpherSettings(); + ParseMorpher(morpherNode, settings, path); + if ( + !fields.TryGetValue("words", out YamlNode wordListNode) || wordListNode is not YamlSequenceNode wordList + ) + { + throw new WordsYamlException(path, $"variant '{name}': expected a 'words' sequence"); + } + List words = wordList.Children.Select(word => ParseWord(word, path)).ToList(); + if ( + words.Count != doc.Words.Count + || !words.Select(w => w.Word).SequenceEqual(doc.Words.Select(w => w.Word), StringComparer.Ordinal) + ) + { + throw new WordsYamlException( + path, + $"variant '{name}': words must contain exactly the default word list in the same order" + ); + } + foreach (WordEntry word in words) + { + if ( + word.BlockedBy.Count > 0 + || word.Neutralizes.Count > 0 + || word.Exercises.Count > 0 + || word.Claims.Count > 0 + || word.Parses.Any(p => p.Exercises.Count > 0) + ) + { + throw new WordsYamlException( + path, + $"variant '{name}', word '{word.Word}': coverage metadata is default-only" + ); + } + } + doc.Variants.Add(new WordRun(name, settings, words)); + } + } + + /// + /// Reads the morpher: mapping into . An empty block, an unknown key, + /// a non-integer or out-of-range count, or a non-boolean flag is an error: a setting the harness + /// cannot apply must not silently fall back to the default it was meant to override. + /// + private static void ParseMorpher(YamlNode node, MorpherSettings settings, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, "front matter.morpher: expected a mapping"); + Dictionary map = RequireMapping(mapNode, MorpherKeys, path, "front matter.morpher"); + if (map.Count == 0) + { + throw new WordsYamlException( + path, + "front matter.morpher: an empty block means the defaults; omit it instead" + ); + } + if (map.TryGetValue("DelReapps", out YamlNode delReappsNode)) + settings.DelReapps = ParseCount(delReappsNode, 0, path, "front matter.morpher.DelReapps"); + if (map.TryGetValue("MaxRoots", out YamlNode maxRootsNode)) + settings.MaxRoots = ParseCount(maxRootsNode, 1, path, "front matter.morpher.MaxRoots"); + if (map.TryGetValue("MergeAnalyses", out YamlNode mergeNode)) + { + settings.MergeAnalyses = ParseBool( + RequireScalarNode(mergeNode, path, "front matter.morpher.MergeAnalyses"), + path + ); + } + if (map.TryGetValue("MakePartialsFinal", out YamlNode partialsNode)) + { + settings.MakePartialsFinal = ParseBool( + RequireScalarNode(partialsNode, path, "front matter.morpher.MakePartialsFinal"), + path + ); + } + if (map.TryGetValue("GuessRoots", out YamlNode guessRootsNode)) + { + settings.GuessRoots = ParseBool( + RequireScalarNode(guessRootsNode, path, "front matter.morpher.GuessRoots"), + path + ); + } + } + + private static int ParseCount(YamlNode node, int minimum, string path, string context) + { + string text = RequireScalarNode(node, path, context); + if ( + !int.TryParse( + text, + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out int value + ) + || value < minimum + ) + { + throw new WordsYamlException( + path, + $"{context}: expected a whole number of at least {minimum}, got '{text}'" + ); + } + return value; + } + + private static WordEntry ParseWord(YamlNode node, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, "each 'words' entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, WordKeys, path, "word"); + + var entry = new WordEntry + { + Word = RequireScalar(map, "word", path, "word"), + Note = RequireScalar(map, "note", path, "word"), + }; + if (map.TryGetValue("provenance", out YamlNode provNode)) + entry.Provenance = RequireScalarNode(provNode, path, $"word '{entry.Word}'.provenance"); + if (map.TryGetValue("expect_fail", out YamlNode failNode)) + entry.ExpectFail = ParseBool(RequireScalarNode(failNode, path, $"word '{entry.Word}'.expect_fail"), path); + if (map.TryGetValue("expect_skip", out YamlNode skipNode)) + entry.ExpectSkip = ParseBool(RequireScalarNode(skipNode, path, $"word '{entry.Word}'.expect_skip"), path); + AppendScalarList(map, "blocked_by", entry.BlockedBy, path, $"word '{entry.Word}'"); + AppendScalarList(map, "neutralizes", entry.Neutralizes, path, $"word '{entry.Word}'"); + AppendScalarList(map, "exercises", entry.Exercises, path, $"word '{entry.Word}'"); + if (map.TryGetValue("claims", out YamlNode claimsNode)) + { + if (claimsNode is not YamlSequenceNode claimsSeq) + throw new WordsYamlException(path, $"word '{entry.Word}'.claims: expected a sequence"); + foreach (YamlNode claimNode in claimsSeq.Children) + entry.Claims.Add(ParseClaim(claimNode, entry.Word, path)); + } + + if (entry.ExpectFail && entry.ExpectSkip) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': 'expect_fail' and 'expect_skip' are mutually exclusive " + + "(a word is either a genuine zero-parse 'ok' or a SKIPPED InvalidShapeException, not both)" + ); + } + if (entry.BlockedBy.Count > 0 && !entry.ExpectFail) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': 'blocked_by' is only meaningful on an 'expect_fail' word" + ); + } + + bool hasParses = map.TryGetValue("parses", out YamlNode parsesNode); + if (entry.ExpectFail || entry.ExpectSkip) + { + if (hasParses) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': expect_fail/expect_skip words must not have 'parses'" + ); + } + } + else + { + if (!hasParses || parsesNode is not YamlSequenceNode parsesSeq || parsesSeq.Children.Count == 0) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': non-expect_fail words must have at least one entry under 'parses'" + ); + } + foreach (YamlNode parseNode in parsesSeq.Children) + entry.Parses.Add(ParseParse(parseNode, entry.Word, path)); + } + + return entry; + } + + private static ParseEntry ParseParse(YamlNode node, string word, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, $"word '{word}': each 'parses' entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, ParseKeys, path, $"word '{word}'.parses[]"); + + var parse = new ParseEntry { Signature = RequireScalar(map, "signature", path, $"word '{word}'.parses[]") }; + if (map.TryGetValue("gloss", out YamlNode glossNode)) + parse.Gloss = RequireScalarNode(glossNode, path, $"word '{word}'.parses[].gloss"); + if (map.TryGetValue("guess", out YamlNode guessNode)) + parse.Guess = ParseBool(RequireScalarNode(guessNode, path, $"word '{word}'.parses[].guess"), path); + if (!map.ContainsKey("rules")) + throw new WordsYamlException(path, $"word '{word}': every parse requires a 'rules' list (may be empty [])"); + AppendScalarList(map, "rules", parse.Rules, path, $"word '{word}'.parses[]"); + AppendScalarList(map, "exercises", parse.Exercises, path, $"word '{word}'.parses[]"); + + return parse; + } + + private static string ParseClaim(YamlNode node, string word, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, $"word '{word}'.claims[]: each entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, ClaimKeys, path, $"word '{word}'.claims[]"); + return RequireScalar(map, "cell", path, $"word '{word}'.claims[]"); + } + + /// + /// Reads a mapping node's keys into a dictionary, hard-failing on any key not in + /// (the "unknown keys are hard errors" rule) and on any + /// non-scalar key (which would indicate a merge key or other non-plain construct). + /// + private static Dictionary RequireMapping( + YamlMappingNode node, + HashSet allowedKeys, + string path, + string context + ) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair entry in node.Children) + { + if (entry.Key is not YamlScalarNode keyScalar || keyScalar.Value == null) + throw new WordsYamlException(path, $"{context}: mapping keys must be plain scalars"); + string key = keyScalar.Value; + if (key == "<<") + throw new WordsYamlException(path, $"{context}: YAML merge keys ('<<') are not allowed"); + if (!allowedKeys.Contains(key)) + { + throw new WordsYamlException( + path, + $"{context}: unknown key '{key}' (allowed: {string.Join(", ", allowedKeys.OrderBy(k => k))})" + ); + } + if (map.ContainsKey(key)) + throw new WordsYamlException(path, $"{context}: duplicate key '{key}'"); + map[key] = entry.Value; + } + return map; + } + + private static string RequireScalar(Dictionary map, string key, string path, string context) + { + if (!map.TryGetValue(key, out YamlNode node)) + throw new WordsYamlException(path, $"{context}: missing required key '{key}'"); + return RequireScalarNode(node, path, $"{context}.{key}"); + } + + private static string RequireScalarNode(YamlNode node, string path, string context) + { + if (node is not YamlScalarNode scalar || scalar.Value == null || scalar.Value.Length == 0) + throw new WordsYamlException(path, $"{context}: expected a non-empty scalar value"); + return scalar.Value; + } + + private static void AppendScalarList( + Dictionary map, + string key, + List target, + string path, + string context + ) + { + if (!map.TryGetValue(key, out YamlNode node)) + return; + if (node is not YamlSequenceNode seq) + throw new WordsYamlException(path, $"{context}.{key}: expected a sequence"); + foreach (YamlNode child in seq.Children) + target.Add(RequireScalarNode(child, path, $"{context}.{key}[]")); + } + + private static bool ParseBool(string value, string path) + { + return value switch + { + "true" => true, + "false" => false, + _ => throw new WordsYamlException(path, $"expected 'true' or 'false', got '{value}'"), + }; + } +} + +public class WordsYamlException(string path, string message, Exception inner = null) + : Exception($"{path}: {message}", inner) { } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs new file mode 100644 index 000000000..510238842 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs @@ -0,0 +1,77 @@ +using ManyConsole; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class BatchCommandTests +{ + [Test] + public void Run_AfterAResumedRun_StartsTheNextRunAtTheFirstWord() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a", "b", "c"]); + try + { + string resumedPath = Path.Combine(directory, "resumed.tsv"); + string freshPath = Path.Combine(directory, "fresh.tsv"); + ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", "--start=2", wordsPath, resumedPath], + TextWriter.Null + ); + ConsoleCommandDispatcher.DispatchCommand(commands, ["batch", wordsPath, freshPath], TextWriter.Null); + + Assert.Multiple(() => + { + Assert.That(File.ReadAllLines(resumedPath)[0], Does.StartWith("2\t")); + Assert.That(File.ReadAllLines(freshPath)[0], Does.StartWith("0\t")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [TestCase("-1")] + [TestCase("x")] + [TestCase("+1")] + public void Run_WithAnInvalidStart_ReportsItWithoutThrowing(string start) + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a"]); + try + { + var console = new StringWriter(); + int result = ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", $"--start={start}", wordsPath, Path.Combine(directory, "out.tsv")], + console + ); + + Assert.Multiple(() => + { + Assert.That(result, Is.Not.EqualTo(0)); + Assert.That(console.ToString(), Does.Contain("--start must be a non-negative integer")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs new file mode 100644 index 000000000..74ee6c2b1 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs @@ -0,0 +1,117 @@ +using System.Reflection; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceCommandLineTests +{ + private static (int ExitCode, string Output, string Error) InvokeProgram(params string[] args) + { + Type programType = typeof(Runner).Assembly.GetType("SIL.Machine.Morphology.HermitCrab.Conformance.Program")!; + MethodInfo main = programType.GetMethod("Main", BindingFlags.Static | BindingFlags.NonPublic)!; + TextWriter originalOut = Console.Out; + TextWriter originalError = Console.Error; + using var output = new StringWriter(); + using var error = new StringWriter(); + try + { + Console.SetOut(output); + Console.SetError(error); + + int exitCode = (int)main.Invoke(null, new object[] { args })!; + return (exitCode, output.ToString(), error.ToString()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + [Test] + [NonParallelizable] + public void CoverageReport_MissingConstructChecklist_Fails() + { + string missing = Path.Combine(Path.GetTempPath(), $"constructs-{Guid.NewGuid():N}.txt"); + + (int exitCode, string output, string error) = InvokeProgram( + "--fixtures", + ConformanceSuite.Root(), + "--coverage-report", + "--constructs", + missing + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(1), error); + Assert.That(output, Does.Contain("CONSTRUCT CHECKLIST NOT FOUND")); + }); + } + + [Test] + [NonParallelizable] + public void CoverageReport_WritesNothingUnderTheFixturesRoot() + { + string suite = ConformanceSuite.Root(); + string[] files = { Path.Combine(suite, "coverage.csv"), Path.Combine(suite, "rules.csv") }; + DateTime[] before = files.Select(File.GetLastWriteTimeUtc).ToArray(); + string[] entriesBefore = Directory.GetFileSystemEntries(suite); + string output = Path.Combine(Path.GetTempPath(), $"hc-coverage-out-{Guid.NewGuid():N}"); + try + { + (int exitCode, string text, string error) = InvokeProgram( + "--fixtures", + suite, + "--coverage-report", + "--coverage-output", + output + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(0), text + error); + Assert.That(files.Select(File.GetLastWriteTimeUtc), Is.EqualTo(before)); + Assert.That(Directory.GetFileSystemEntries(suite), Is.EquivalentTo(entriesBefore)); + foreach (string file in files) + { + Assert.That( + File.ReadAllText(Path.Combine(output, Path.GetFileName(file))).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(file).ReplaceLineEndings("\n")), + Path.GetFileName(file) + ); + } + }); + } + finally + { + if (Directory.Exists(output)) + Directory.Delete(output, recursive: true); + } + } + + [Test] + [NonParallelizable] + public void CoverageReport_RefusesAnOutputInsideTheFixturesRoot() + { + string suite = ConformanceSuite.Root(); + string inside = Path.Combine(suite, $"hc-coverage-out-{Guid.NewGuid():N}"); + + (int exitCode, string _, string error) = InvokeProgram( + "--fixtures", + suite, + "--coverage-report", + "--coverage-output", + inside + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(2), error); + Assert.That(error, Does.Contain("not the suite clone")); + Assert.That(Directory.Exists(inside), Is.False); + }); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs new file mode 100644 index 000000000..3eb24f37c --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs @@ -0,0 +1,353 @@ +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Runs every conformance fixture inside the ordinary test suite, so a change to the engine, the runner or +/// the suite pin that breaks a fixture fails dotnet test as well as the standalone hc-conformance CLI. +/// +[TestFixture] +public sealed class ConformanceFixtureGateTests +{ + /// Guards against the suite silently shrinking to nothing and still reporting green. + private const int MinimumFixtures = 25; + + private static List Discover() => Fixture.DiscoverAll(ConformanceSuite.Root()); + + [Test] + public void EveryConformanceFixturePassesSelfCheck() + { + List fixtures = Discover(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(MinimumFixtures)); + + RunReport report = Runner.RunSelfCheck( + fixtures, + includePathological: false, + Runner.DefaultCapabilities, + propose: false, + TextWriter.Null + ); + + string[] failures = report + .Results.Where(result => result.Outcome == FixtureOutcome.Failed) + .Select(result => $"{result.FixtureId}: {result.Reason}") + .OrderBy(line => line, StringComparer.Ordinal) + .ToArray(); + + Assert.That(failures, Is.Empty, $"conformance fixtures failed:\n {string.Join("\n ", failures)}"); + Assert.That( + report.Passed, + Is.GreaterThanOrEqualTo(MinimumFixtures), + "the run must actually execute fixtures, not skip them all" + ); + } + + [Test] + public void EveryBlockedByLabelIsARuleIdInItsFixtureGrammar() + { + List fixtures = Discover(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(MinimumFixtures)); + + var unknown = new List(); + foreach (Fixture fixture in fixtures) + { + GrammarRuleIndex index = GrammarRuleIndex.Load(fixture.GrammarPath); + unknown.AddRange(CoverageReport.FindUnknownBlockedBy(fixture.Id, fixture.Words, index.AllRuleIds)); + } + + Assert.That( + unknown.Select(u => $"{u.FixtureId}: {u.Word} blocked_by {u.Label}"), + Is.Empty, + "blocked_by labels must name a rule id declared in the fixture's grammar.xml" + ); + } + + [Test] + public void CheckedInCoverageTablesAreUpToDate() + { + string root = ConformanceSuite.Root(); + string temp = Path.Combine(Path.GetTempPath(), $"hc-coverage-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temp); + try + { + string coverage = Path.Combine(temp, "coverage.csv"); + string rules = Path.Combine(temp, "rules.csv"); + CoverageReport.WriteCsvs(Discover(), coverage, rules); + + Assert.Multiple(() => + { + Assert.That( + File.ReadAllText(coverage).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(Path.Combine(root, "coverage.csv")).ReplaceLineEndings("\n")), + "regenerate in a clone of the suite with: hc-conformance --fixtures conformance --coverage-report --coverage-output " + ); + Assert.That( + File.ReadAllText(rules).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(Path.Combine(root, "rules.csv")).ReplaceLineEndings("\n")), + "regenerate in a clone of the suite with: hc-conformance --fixtures conformance --coverage-report --coverage-output " + ); + }); + } + finally + { + Directory.Delete(temp, recursive: true); + } + } + + [Test] + public void EveryFixtureContributesAtLeastOneCoverageRow() + { + var noExerciseByDesign = new HashSet(StringComparer.Ordinal) + { + "edge-cases/simultaneous-epenthesis-cascade", + "edge-cases/truncate-morphotactic-copy-u", + }; + + List fixtures = Discover(); + Assert.That( + fixtures, + Has.Count.GreaterThanOrEqualTo(MinimumFixtures), + "the discovery walk must find fixtures, not silently see zero" + ); + + HashSet discoveredIds = fixtures.Select(f => f.Id).ToHashSet(StringComparer.Ordinal); + string[] staleAllowlistEntries = noExerciseByDesign + .Where(id => !discoveredIds.Contains(id)) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + Assert.That( + staleAllowlistEntries, + Is.Empty, + "these allowlist entries name no discovered fixture (renamed or typo'd?):\n " + + string.Join("\n ", staleAllowlistEntries) + ); + + string temp = Path.Combine(Path.GetTempPath(), $"hc-exercise-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temp); + try + { + string coverage = Path.Combine(temp, "coverage.csv"); + string rules = Path.Combine(temp, "rules.csv"); + CoverageReport.WriteCsvs(fixtures, coverage, rules); + + string[] lines = File.ReadAllLines(coverage); + Assert.That( + lines, + Has.Length.GreaterThan(1), + "coverage.csv must have a header plus at least one data row -- an empty table proves nothing" + ); + + HashSet languagesWithRows = lines + .Skip(1) + .Select(line => line[..line.IndexOf(',')]) + .ToHashSet(StringComparer.Ordinal); + + string[] silent = fixtures + .Where(f => !languagesWithRows.Contains(f.Words.Language) && !noExerciseByDesign.Contains(f.Id)) + .Select(f => f.Id) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + silent, + Is.Empty, + "these fixtures contribute ZERO rows to coverage.csv (no 'exercises:' tag anywhere) and " + + $"are not in the named allowlist above:\n {string.Join("\n ", silent)}" + ); + } + finally + { + Directory.Delete(temp, recursive: true); + } + } + + // Every construct a fixture claims must be a real line in the checklist, and every checklist line + // must be claimed. parity-check.py enforces the second half; nothing enforced the first. + [Test] + public void EveryExercisedConstructIsDeclaredInTheChecklist() + { + string root = ConformanceSuite.Root(); + var declared = CoverageReport + .LoadConstructChecklist(Path.Combine(root, "constructs.txt")) + .ToHashSet(StringComparer.Ordinal); + + string[] unknown = Discover() + .SelectMany(fixture => fixture.Words.Words) + .SelectMany(word => word.Exercises.Concat(word.Parses.SelectMany(parse => parse.Exercises))) + .Distinct(StringComparer.Ordinal) + .Where(construct => !declared.Contains(construct)) + .OrderBy(construct => construct, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + unknown, + Is.Empty, + $"add these to constructs.txt or fix the tag:\n {string.Join("\n ", unknown)}" + ); + } + + [Test] + public void FixturesWithRulesAttributeAtLeastOneParseToThem() + { + var silent = new List(); + foreach (Fixture fixture in Discover()) + { + bool declaresRules = File.ReadAllText(fixture.GrammarPath) + .Contains("MorphologicalRule id=", StringComparison.Ordinal); + bool attributesAny = fixture.Words.Words.Any(word => word.Parses.Any(parse => parse.Rules.Count > 0)); + if (declaresRules && !attributesAny) + silent.Add(fixture.Id); + } + + Assert.That( + silent, + Is.Empty, + "these declare morphological rules but no parse names one, so nothing they contain can be trace-verified" + ); + } + + [Test] + public void EveryFixtureIsMarkedFieldWorksProducible() + { + string[] notProducible = Discover() + .Where(fixture => fixture.Words.FieldworksProducible != true) + .Select(fixture => fixture.Id) + .ToArray(); + + Assert.That( + notProducible, + Is.Empty, + "conformance keeps only grammars a FieldWorks project can produce (PROTOCOL.md section 9); rewrite or delete these" + ); + } + + [Test] + public void FieldWorksWitnessDirectoriesAreShapedCorrectly() + { + var allowedManifestKeys = new HashSet(StringComparer.Ordinal) + { + "version", + "base_sha256", + "cases", + "id", + "operations", + "expect", + "remove_phoneme", + "remove_all_phonemes", + "guid", + "assert_representations", + "require_unreferenced", + "xample_projection", + "hc_analyses", + "inferred_segments", + }; + + var errors = new List(); + foreach (Fixture fixture in Discover()) + { + string fieldworksDir = Path.Combine(fixture.Directory, "fieldworks"); + if (!Directory.Exists(fieldworksDir)) + continue; + + if (fixture.Words.FieldworksProducible != true) + { + errors.Add($"{fixture.Id}: has a fieldworks/ directory but is not 'fieldworks_producible: true'"); + continue; + } + + var expectedEntries = new HashSet(StringComparer.Ordinal) + { + "project.fwdata", + "phonology-mutations.yaml", + "WritingSystemStore", + }; + string[] actualEntries = Directory.GetFileSystemEntries(fieldworksDir).Select(Path.GetFileName).ToArray()!; + foreach (string extra in actualEntries.Where(e => !expectedEntries.Contains(e!))) + errors.Add($"{fixture.Id}: fieldworks/ contains an unexpected entry '{extra}'"); + foreach (string missing in expectedEntries.Where(e => !actualEntries.Contains(e))) + errors.Add($"{fixture.Id}: fieldworks/ is missing '{missing}'"); + + string writingSystemDir = Path.Combine(fieldworksDir, "WritingSystemStore"); + if (Directory.Exists(writingSystemDir)) + { + string[] nonLdml = Directory + .GetFiles(writingSystemDir) + .Select(Path.GetFileName) + .Where(name => !name!.EndsWith(".ldml", StringComparison.OrdinalIgnoreCase)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray()!; + if (nonLdml.Length > 0) + { + errors.Add( + $"{fixture.Id}: WritingSystemStore/ contains non-.ldml entries: {string.Join(", ", nonLdml)}" + ); + } + } + + string projectPath = Path.Combine(fieldworksDir, "project.fwdata"); + string manifestPath = Path.Combine(fieldworksDir, "phonology-mutations.yaml"); + if (!File.Exists(projectPath) || !File.Exists(manifestPath)) + continue; // already recorded as missing above + + string manifestText = File.ReadAllText(manifestPath); + + Match versionMatch = Regex.Match(manifestText, @"^version:\s*(\d+)\s*$", RegexOptions.Multiline); + if (!versionMatch.Success || versionMatch.Groups[1].Value != "1") + errors.Add($"{fixture.Id}: phonology-mutations.yaml has no 'version: 1'"); + + Match shaMatch = Regex.Match(manifestText, @"^base_sha256:\s*([0-9a-f]{64})\s*$", RegexOptions.Multiline); + if (!shaMatch.Success) + { + errors.Add($"{fixture.Id}: phonology-mutations.yaml has no 64-character lowercase-hex 'base_sha256'"); + } + else + { + using var sha256 = SHA256.Create(); + using FileStream stream = File.OpenRead(projectPath); + string actual = Convert.ToHexString(sha256.ComputeHash(stream)).ToLowerInvariant(); + if (!string.Equals(actual, shaMatch.Groups[1].Value, StringComparison.Ordinal)) + { + errors.Add( + $"{fixture.Id}: phonology-mutations.yaml base_sha256 {shaMatch.Groups[1].Value} " + + $"does not match project.fwdata's actual sha256 {actual}" + ); + } + } + + foreach ( + Match keyMatch in Regex.Matches( + manifestText, + @"^\s*-?\s*([A-Za-z_][A-Za-z0-9_]*):", + RegexOptions.Multiline + ) + ) + { + string key = keyMatch.Groups[1].Value; + if (!allowedManifestKeys.Contains(key)) + errors.Add($"{fixture.Id}: phonology-mutations.yaml uses a key outside the v1 vocabulary: '{key}'"); + } + + foreach ( + Match valueMatch in Regex.Matches( + manifestText, + @"(xample_projection|hc_analyses):\s*(\S+)\s*$", + RegexOptions.Multiline + ) + ) + { + if (valueMatch.Groups[2].Value != "same_as_base") + { + errors.Add( + $"{fixture.Id}: phonology-mutations.yaml has {valueMatch.Groups[1].Value}: " + + $"{valueMatch.Groups[2].Value}, but only 'same_as_base' is defined" + ); + } + } + } + + Assert.That(errors, Is.Empty, string.Join("\n ", errors)); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs new file mode 100644 index 000000000..697b33514 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs @@ -0,0 +1,211 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceHarnessTests +{ + private sealed class AlwaysCrashingEngine(IEnumerable capabilities) : IEngine + { + public string Name => "always-crashing"; + public IReadOnlySet Capabilities { get; } = new HashSet(capabilities, StringComparer.Ordinal); + + public List Run(MaterializedFixture fixture) => + throw new EngineCrashException($"stub crash for '{fixture.Id}'"); + } + + // A copy of the slow fixture with a 1 ms budget, so the self-check watchdog fires after its 1 s margin while + // the fast first word has already finished and the 924-parse second word has not. + private static FixtureResult RunSlowFixtureOverBudget(string extraFrontMatter) + { + string root = Path.Combine(Path.GetTempPath(), $"slow-fixture-{Guid.NewGuid():N}"); + string source = Path.Combine(ConformanceSuite.Root(), "edge-cases", "deep-optional-affix-nesting"); + string target = Path.Combine(root, "edge-cases", "deep-optional-affix-nesting"); + Directory.CreateDirectory(target); + foreach (string file in Directory.GetFiles(source)) + File.Copy(file, Path.Combine(target, Path.GetFileName(file))); + string wordsPath = Path.Combine(target, "words.yaml"); + string words = File.ReadAllText(wordsPath); + Assert.That(words, Does.Contain("budget_ms: 15000")); + File.WriteAllText(wordsPath, words.Replace("budget_ms: 15000", "budget_ms: 1" + extraFrontMatter)); + try + { + RunReport report = Runner.RunSelfCheck( + Fixture.DiscoverAll(root), + includePathological: true, + engineCapabilities: new HashSet(StringComparer.Ordinal), + propose: false, + proposeOutput: TextWriter.Null + ); + return report.Results.Single(); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + private static List OneRow(string signature) => [new TsvRow("bat", "ok", signature)]; + + [Test] + public void Diff_ClassMembersInDifferentOrder_Passes() + { + FixtureDiff diff = Diff.Compare(OneRow("|b[ae]t"), OneRow("|b[ea]t")); + + Assert.That(diff.AllPassed, Is.True); + } + + [Test] + public void Diff_DifferentClassMembers_Fails() + { + FixtureDiff diff = Diff.Compare(OneRow("|b[ae]t"), OneRow("|b[ao]t")); + + Assert.That(diff.AllPassed, Is.False); + } + + [Test] + public void Diff_NegatedClassMembersInDifferentOrder_PassesButNegationStillMatters() + { + Assert.That(Diff.Compare(OneRow("|b[^ae]t"), OneRow("|b[^ea]t")).AllPassed, Is.True); + Assert.That(Diff.Compare(OneRow("|b[^ae]t"), OneRow("|b[ae]t")).AllPassed, Is.False); + } + + [Test] + public void CompareRuleTraceMultisets_SameSignatureWithDistinctTraces_PassesRegardlessOfOrder() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead", "crShared" }), + (Signature, new[] { "crRightHead" }), + }; + var actual = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crRightHead" }), + (Signature, new[] { "crShared", "crLeftHead" }), + }; + + Assert.That(Runner.CompareRuleTraceMultisets(expected, actual), Is.Empty); + } + + [Test] + public void CompareRuleTraceMultisets_WrongTraceForSameSignature_Fails() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] { (Signature, new[] { "crLeftHead" }) }; + var actual = new (string Signature, IEnumerable RuleIds)[] { (Signature, new[] { "crWrongHead" }) }; + + List details = Runner.CompareRuleTraceMultisets(expected, actual); + + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details, Has.Some.Contains("rules [crLeftHead]: declared 1, traced 0")); + Assert.That(details, Has.Some.Contains("rules [crWrongHead]: declared 0, traced 1")); + } + + [Test] + public void CompareRuleTraceMultisets_DuplicateTraceMultiplicityMustMatch() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead" }), + (Signature, new[] { "crLeftHead" }), + }; + var actual = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead" }), + (Signature, new[] { "crRightHead" }), + }; + + List details = Runner.CompareRuleTraceMultisets(expected, actual); + + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details, Has.Some.Contains("rules [crLeftHead]: declared 2, traced 1")); + Assert.That(details, Has.Some.Contains("rules [crRightHead]: declared 0, traced 1")); + } + + [Test] + public void CompareRuleTraceMultisets_BracketClassMembersInDifferentOrder_Passes() + { + var expected = new (string Signature, IEnumerable RuleIds)[] { ("|b[ea]t", new[] { "crLeftHead" }) }; + var actual = new (string Signature, IEnumerable RuleIds)[] { ("|b[ae]t", new[] { "crLeftHead" }) }; + + Assert.That(Runner.CompareRuleTraceMultisets(expected, actual), Is.Empty); + } + + [Test] + public void RunSelfCheck_ExpectCrashFixtureThatTimesOut_Fails() + { + FixtureResult result = RunSlowFixtureOverBudget("\nexpect_crash: true"); + + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + } + + [Test] + public void RunSelfCheck_TimedOutFixture_ReportsNoWordResults() + { + FixtureResult result = RunSlowFixtureOverBudget(""); + + Assert.Multiple(() => + { + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + Assert.That(result.WordResults, Is.Empty); + }); + } + + [Test] + public void RunAdapter_ExpectCrashFixture_FailsWhenTheEngineAlsoCrashesOnAnOrdinaryFixture() + { + List all = Fixture.DiscoverAll(ConformanceSuite.Root()); + Fixture crash = all.First(fixture => fixture.Words.ExpectCrash); + Fixture ordinary = all.First(fixture => !fixture.Words.ExpectCrash && fixture.Words.BudgetMs is null); + var engine = new AlwaysCrashingEngine(crash.Words.Requires.Concat(ordinary.Words.Requires)); + + RunReport report = Runner.RunAdapter(new List { crash, ordinary }, engine, includePathological: true); + + FixtureResult crashResult = report.Results.Single(result => result.FixtureId == crash.Id); + Assert.That(crashResult.Outcome, Is.EqualTo(FixtureOutcome.Failed), crashResult.Reason); + Assert.That(crashResult.Reason, Does.Contain(ordinary.Id)); + } + + [Test] + public void WordsYamlLoader_NonNumericBudget_ThrowsWordsYamlException() + { + string path = Path.Combine(Path.GetTempPath(), $"words-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(path, "language: Probe\nbudget_ms: 5s\nwords: []\n"); + try + { + WordsYamlException ex = Assert.Throws(() => WordsYamlLoader.Load(path))!; + Assert.That(ex.Message, Does.Contain("budget_ms")); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void GrammarRuleIndex_TwoRulesWithTheSameName_Throws() + { + string path = Path.Combine(Path.GetTempPath(), $"grammar-{Guid.NewGuid():N}.xml"); + File.WriteAllText( + path, + """ + + Epenthesis + Epenthesis + + """ + ); + try + { + InvalidDataException ex = Assert.Throws(() => GrammarRuleIndex.Load(path))!; + Assert.That(ex.Message, Does.Contain("pr1").And.Contain("pr2")); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs new file mode 100644 index 000000000..db3c8f2f2 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs @@ -0,0 +1,302 @@ +using System.Globalization; +using System.Text.Json; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceManifestTests +{ + private static string SuiteRoot() => ConformanceSuite.Root(); + + private static ConformanceManifest Manifest() => ConformanceManifestGenerator.Generate(SuiteRoot()); + + /// A path as the manifest names it: relative to the suite root, the conformance suite. + private static string InSuite(string manifestPath) => + Path.Combine(SuiteRoot(), manifestPath.Replace('/', Path.DirectorySeparatorChar)); + + private const string LibraryDtdRelativePath = "src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd"; + + private static string LibraryDtdPath => + Path.Combine( + ConformanceSuite.RepositoryRoot(), + LibraryDtdRelativePath.Replace('/', Path.DirectorySeparatorChar) + ); + + [Test] + public void FormatIdentifiersAreExact() + { + ConformanceManifest manifest = Manifest(); + + Assert.Multiple(() => + { + Assert.That(manifest.FormatVersion, Is.EqualTo("hc-conformance-manifest/v2")); + Assert.That(manifest.WordsAuthoringFormatVersion, Is.EqualTo("hc-conformance-words/v1")); + Assert.That(manifest.GrammarFormatVersion, Is.EqualTo("sil-machine-hermit-crab-input-xml/v1")); + Assert.That(manifest.SourceHash, Has.Length.EqualTo(64)); + Assert.That(manifest.DtdSha256, Has.Length.EqualTo(64)); + Assert.That(manifest.Fixtures, Is.Not.Empty); + }); + } + + // The suite is consumed on its own, so every path the manifest names is relative to the suite root + // and carries no directory prefix. + [Test] + public void EveryPathTheManifestNamesIsRelativeToTheSuiteRoot() + { + ConformanceManifest manifest = Manifest(); + + Assert.Multiple(() => + { + Assert.That(manifest.DtdPath, Is.EqualTo("HermitCrabInput.dtd")); + Assert.That(File.Exists(InSuite(manifest.DtdPath)), Is.True, manifest.DtdPath); + foreach (ManifestFixture fixture in manifest.Fixtures) + { + Assert.That(fixture.GrammarPath, Is.EqualTo($"{fixture.FixtureId}/grammar.xml")); + Assert.That(fixture.WordsPath, Is.EqualTo($"{fixture.FixtureId}/words.yaml")); + Assert.That(File.Exists(InSuite(fixture.GrammarPath)), Is.True, fixture.GrammarPath); + Assert.That(File.Exists(InSuite(fixture.WordsPath)), Is.True, fixture.WordsPath); + Assert.That(fixture.GrammarPath, Does.Not.Contain("\\")); + Assert.That(fixture.GrammarSha256, Has.Length.EqualTo(64)); + Assert.That(fixture.WordsSha256, Has.Length.EqualTo(64)); + Assert.That(fixture.DisplayLanguage, Is.Not.Empty, fixture.FixtureId); + Assert.That(fixture.CaseCount, Is.GreaterThan(0), fixture.FixtureId); + Assert.That( + fixture.Category, + Is.EqualTo( + fixture.FixtureId.StartsWith("languages/", StringComparison.Ordinal) ? "language" : "edge-case" + ) + ); + } + }); + } + + // The library reads its DTD as an embedded resource and cannot move; the suite's copy is the + // authority. They must not drift. + [Test] + public void PublishedDtdMatchesTheLibraryResource() + { + Assert.That( + File.ReadAllText(InSuite(ConformanceManifestGenerator.DtdRelativePath)).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(LibraryDtdPath).ReplaceLineEndings("\n")), + $"the suite's {ConformanceManifestGenerator.DtdRelativePath} is authoritative: change it in the suite, bump " + + $"conformance.commit and copy it over {LibraryDtdRelativePath}" + ); + } + + [Test] + public void CrashFixturesDeclareExactlyOneCase() + { + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures.Where(item => item.ExpectedCrash)) + Assert.That(fixture.CaseCount, Is.EqualTo(1), fixture.FixtureId); + }); + } + + // A regenerated manifest must be byte-identical, or `--check-manifest` cannot detect drift. + [Test] + public void GenerationIsDeterministicAndCanonical() + { + string first = ManifestJson.Serialize(Manifest()); + string second = ManifestJson.Serialize(Manifest()); + + Assert.That(second, Is.EqualTo(first)); + Assert.That(() => JsonDocument.Parse(first), Throws.Nothing); + } + + [Test] + public void TheCheckedInManifestMatchesRegeneration() + { + string path = InSuite(ConformanceManifestGenerator.ManifestRelativePath); + const string Fix = "regenerate it in the suite with: python3 scripts/generate_manifest.py"; + + Assert.That(File.Exists(path), Is.True, Fix); + Assert.That( + File.ReadAllText(path).ReplaceLineEndings("\n"), + Is.EqualTo(ManifestJson.Serialize(Manifest()) + "\n"), + Fix + ); + } + + [Test] + public void TheManifestValidatesAgainstItsPublishedSchema() + { + Json.Schema.JsonSchema schema = Json.Schema.JsonSchema.FromFile( + InSuite("schema/conformance-manifest.schema.json") + ); + using JsonDocument document = JsonDocument.Parse(ManifestJson.Serialize(Manifest())); + + Json.Schema.EvaluationResults results = schema.Evaluate( + document.RootElement, + new Json.Schema.EvaluationOptions { OutputFormat = Json.Schema.OutputFormat.List } + ); + + IEnumerable failures = (results.Details ?? new List()) + .Where(detail => detail.Errors is { Count: > 0 }) + .SelectMany(detail => detail.Errors!.Select(error => $"{detail.InstanceLocation}: {error.Value}")) + .Take(20); + + Assert.That(results.IsValid, Is.True, string.Join(Environment.NewLine, failures)); + } + + // words.yaml is the canonical authored format, so the published schema has to describe every + // file the loader already accepts. + [Test] + public void EveryWordsFileValidatesAgainstThePublishedSchema() + { + string schema = InSuite(WordsSchemaValidation.SchemaRelativePath); + + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures) + { + IReadOnlyList violations = WordsSchemaValidation.Validate(InSuite(fixture.WordsPath), schema); + Assert.That(violations, Is.Empty, $"{fixture.FixtureId}: {string.Join(" | ", violations.Take(6))}"); + } + }); + } + + [TestCase("anchors", "language: X\nwords: &a []\n")] + [TestCase("merge keys", "language: X\nwords:\n - <<: {word: a}\n note: n\n")] + public void PlainYamlOnlyIsEnforced(string _, string yaml) + { + string schema = InSuite(WordsSchemaValidation.SchemaRelativePath); + string path = Path.Combine(Path.GetTempPath(), $"hc-words-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(path, yaml); + try + { + Assert.That(() => WordsSchemaValidation.Validate(path, schema), Throws.TypeOf()); + } + finally + { + File.Delete(path); + } + } + + /// A minimal suite carrying one real fixture, for mutation tests. + private static string CreateMiniSuite(string fixtureId) + { + string source = SuiteRoot(); + string root = Path.Combine(Path.GetTempPath(), $"hc-manifest-suite-{Guid.NewGuid():N}"); + + void Copy(string relative) + { + string target = Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(Path.Combine(source, relative.Replace('/', Path.DirectorySeparatorChar)), target); + } + + Copy(ConformanceManifestGenerator.DtdRelativePath); + Copy("constructs.txt"); + Copy("parity-check.py"); + Copy("schema/words.schema.json"); + Copy("schema/conformance-manifest.schema.json"); + Copy($"{fixtureId}/grammar.xml"); + Copy($"{fixtureId}/words.yaml"); + return root; + } + + // Every input the manifest rests on must move its sourceHash, or a stale manifest cannot be + // told from a current one. + [TestCase("{0}/grammar.xml")] + [TestCase("{0}/words.yaml")] + [TestCase("constructs.txt")] + [TestCase("parity-check.py")] + [TestCase("schema/words.schema.json")] + public void EverySourceFileMovesTheSourceHash(string relativeTemplate) + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = CreateMiniSuite(FixtureId); + try + { + string before = ConformanceManifestGenerator.Generate(root).SourceHash; + string target = Path.Combine( + root, + string.Format(CultureInfo.InvariantCulture, relativeTemplate, FixtureId) + .Replace('/', Path.DirectorySeparatorChar) + ); + File.AppendAllText(target, "\n# drift\n"); + + Assert.That( + ConformanceManifestGenerator.Generate(root).SourceHash, + Is.Not.EqualTo(before), + relativeTemplate + ); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + // Fixture identity comes from the two declared roots only; a stray directory elsewhere under + // the suite root must not be able to present itself as a fixture. + [Test] + public void DiscoveryAcceptsOnlyTheTwoDeclaredFixtureRoots() + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = CreateMiniSuite(FixtureId); + try + { + string legacy = Path.Combine(root, "allomorphy", "strrep-identity"); + Directory.CreateDirectory(legacy); + foreach (string name in new[] { "grammar.xml", "words.yaml" }) + { + File.Copy(Path.Combine(root, "edge-cases", "strrep-identity", name), Path.Combine(legacy, name)); + } + + Assert.That( + ConformanceManifestGenerator.Generate(root).Fixtures.Select(fixture => fixture.FixtureId), + Is.EqualTo(new[] { FixtureId }) + ); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public void EveryGrammarValidatesAgainstThePublishedDtd() + { + string dtd = InSuite(ConformanceManifestGenerator.DtdRelativePath); + + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures) + { + Assert.That( + GrammarValidation.Validate(InSuite(fixture.GrammarPath), dtd), + Is.Empty, + $"{fixture.FixtureId}: grammar must validate against the published DTD" + ); + } + }); + } + + [Test] + public void TheResolverAdmitsOnlyThePinnedDtdSystemIdentifier() + { + string dtd = InSuite(ConformanceManifestGenerator.DtdRelativePath); + string directory = Path.Combine(Path.GetTempPath(), $"hc-manifest-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + string hostile = Path.Combine(directory, "grammar.xml"); + File.WriteAllText( + hostile, + "\n" + + "\n" + + "\n" + ); + + Assert.That(() => GrammarValidation.Validate(hostile, dtd), Throws.TypeOf()); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs new file mode 100644 index 000000000..a431ce8b1 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs @@ -0,0 +1,38 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceMorpherFactoryTests +{ + [Test] + public void DefaultMorpherIsSequentialAndUntraced() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + + Morpher morpher = ConformanceMorpherFactory.Create(language, fixture.Words.Morpher); + + Assert.Multiple(() => + { + Assert.That(morpher.MaxDegreeOfParallelism, Is.EqualTo(1)); + Assert.That(morpher.TraceManager.IsTracing, Is.False); + }); + } + + [Test] + public void TracingMorpherCanBeCreatedSequentially() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + + Morpher morpher = ConformanceMorpherFactory.CreateTracing(language, fixture.Words.Morpher); + + Assert.Multiple(() => + { + Assert.That(morpher.MaxDegreeOfParallelism, Is.EqualTo(1)); + Assert.That(morpher.TraceManager.IsTracing, Is.True); + }); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs new file mode 100644 index 000000000..431cd67f3 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs @@ -0,0 +1,93 @@ +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// Finds the conformance suite for the tests that read it: the clone in conformance/ at the commit in +/// conformance.commit, or the directory named by HC_CONFORMANCE_SUITE. +internal static class ConformanceSuite +{ + public const string OverrideVariable = "HC_CONFORMANCE_SUITE"; + private const string SuiteUrl = "https://github.com/sillsdev/fieldworks-parser-conformance.git"; + + private static string? VerifiedClone; + + /// The suite directory. Fails the running test, naming the command that fixes it, when the clone is + /// missing or is not at the pinned commit. A suite named by HC_CONFORMANCE_SUITE is used as it is. + public static string Root() + { + string? overridden = Environment.GetEnvironmentVariable(OverrideVariable); + if (!string.IsNullOrWhiteSpace(overridden)) + { + string directory = Path.GetFullPath(overridden); + if (!File.Exists(Path.Combine(directory, "constructs.txt"))) + Assert.Fail($"{OverrideVariable}={directory} is not the conformance suite: it has no constructs.txt."); + return directory; + } + + if (VerifiedClone is not null) + return VerifiedClone; + + string repositoryRoot = RepositoryRoot(); + string suite = Path.Combine(repositoryRoot, "conformance"); + string pinFile = Path.Combine(repositoryRoot, "conformance.commit"); + string pin = File.Exists(pinFile) ? File.ReadAllText(pinFile).Trim() : ""; + string manual = + $"{Environment.NewLine} git clone --filter=blob:none {SuiteUrl} conformance" + + $"{Environment.NewLine} git -C conformance checkout --detach {pin}"; + + if (!File.Exists(Path.Combine(suite, "constructs.txt"))) + { + Assert.Fail($"The conformance suite is missing: {suite} does not hold it. Run these two commands:{manual}"); + return suite; + } + + ProcessStartInfo startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = suite; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.ArgumentList.Add("rev-parse"); + startInfo.ArgumentList.Add("HEAD"); + string head; + try + { + using Process process = Process.Start(startInfo)!; + head = process.StandardOutput.ReadToEnd().Trim(); + process.WaitForExit(); + if (process.ExitCode != 0) + head = ""; + } + catch (System.ComponentModel.Win32Exception) + { + head = ""; + } + + if (head != pin) + { + Assert.Fail( + $"The conformance suite in {suite} is at '{head}', not the commit in conformance.commit ({pin}). " + + $"Run these two commands:{Environment.NewLine} git -C conformance fetch origin" + + $"{Environment.NewLine} git -C conformance checkout --detach {pin}" + ); + } + + VerifiedClone = suite; + return suite; + } + + /// The directory holding Machine.sln; it does not need the suite. + public static string RepositoryRoot() + { + string? directory = TestContext.CurrentContext.TestDirectory; + while (directory is not null) + { + if (File.Exists(Path.Combine(directory, "Machine.sln"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root (the directory holding Machine.sln)."); + return string.Empty; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs new file mode 100644 index 000000000..66d6c6386 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs @@ -0,0 +1,156 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class DeadRuleAllowlistTests +{ + private static CoverageReport.DeadRule Dead(string fixtureId, string ruleId) => + new() { FixtureId = fixtureId, RuleId = ruleId }; + + private static string WriteAllowlist(params string[] lines) + { + string path = Path.GetTempFileName(); + File.WriteAllLines(path, lines); + return path; + } + + [Test] + public void AllowedDeadRulesPass() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "r1")] = "decoy" }; + + var (unexpected, stale) = CoverageReport.CheckDeadRules(new[] { Dead("edge-cases/a", "r1") }, allowed); + + Assert.That(unexpected, Is.Empty); + Assert.That(stale, Is.Empty); + } + + [Test] + public void DeadRuleMissingFromAllowlistIsUnexpected() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "r1")] = "decoy" }; + + var (unexpected, _) = CoverageReport.CheckDeadRules( + new[] { Dead("edge-cases/a", "r1"), Dead("edge-cases/a", "r2") }, + allowed + ); + + Assert.That(unexpected.Select(d => d.RuleId), Is.EqualTo(new[] { "r2" })); + } + + [Test] + public void AllowedRuleThatIsNoLongerDeadIsStale() + { + var allowed = new Dictionary<(string, string), string> + { + [("edge-cases/a", "r1")] = "decoy", + [("edge-cases/b", "r1")] = "decoy", + }; + + var (_, stale) = CoverageReport.CheckDeadRules(new[] { Dead("edge-cases/a", "r1") }, allowed); + + Assert.That(stale, Is.EqualTo(new[] { ("edge-cases/b", "r1") })); + } + + [Test] + public void RuleIdsMatchPerFixtureAndOrdinally() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "rule")] = "decoy" }; + + var (unexpected, stale) = CoverageReport.CheckDeadRules( + new[] { Dead("edge-cases/b", "rule"), Dead("edge-cases/a", "Rule") }, + allowed + ); + + Assert.That(unexpected, Has.Count.EqualTo(2)); + Assert.That(stale, Has.Count.EqualTo(1)); + } + + [Test] + public void LoaderSkipsCommentsAndBlankLines() + { + string path = WriteAllowlist("# header", "", "edge-cases/a\tr1\tdecoy"); + try + { + Dictionary<(string FixtureId, string RuleId), string> allowed = CoverageReport.LoadDeadRuleAllowlist(path); + + Assert.That(allowed[("edge-cases/a", "r1")], Is.EqualTo("decoy")); + Assert.That(allowed, Has.Count.EqualTo(1)); + } + finally + { + File.Delete(path); + } + } + + [TestCase("edge-cases/a\tr1")] + [TestCase("edge-cases/a\tr1\t ")] + [TestCase("edge-cases/a\tr1\tdecoy\textra")] + public void LoaderRejectsRowsWithoutAReason(string row) + { + string path = WriteAllowlist(row); + try + { + Assert.Throws(() => CoverageReport.LoadDeadRuleAllowlist(path)); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void LoaderRejectsDuplicateEntries() + { + string path = WriteAllowlist("edge-cases/a\tr1\tdecoy", "edge-cases/a\tr1\tagain"); + try + { + Assert.Throws(() => CoverageReport.LoadDeadRuleAllowlist(path)); + } + finally + { + File.Delete(path); + } + } + + private static WordsYaml WordsBlockedBy(params string[] labels) + { + var words = new WordsYaml { Language = "L" }; + var word = new WordEntry { Word = "w", ExpectFail = true }; + word.BlockedBy.AddRange(labels); + words.Words.Add(word); + return words; + } + + [Test] + public void BlockedByLabelThatIsAGrammarRuleIdPasses() + { + var unknown = CoverageReport.FindUnknownBlockedBy("edge-cases/a", WordsBlockedBy("r1"), new[] { "r1", "r2" }); + + Assert.That(unknown, Is.Empty); + } + + [Test] + public void BlockedByLabelThatIsNotAGrammarRuleIdFails() + { + var unknown = CoverageReport.FindUnknownBlockedBy( + "edge-cases/a", + WordsBlockedBy("r1", "bogus"), + new[] { "r1", "r2" } + ); + + Assert.That(unknown.Select(u => u.Label), Is.EqualTo(new[] { "bogus" })); + Assert.That(unknown[0].FixtureId, Is.EqualTo("edge-cases/a")); + Assert.That(unknown[0].Word, Is.EqualTo("w")); + } + + [Test] + public void BlockedByLabelsAreComparedOrdinally() + { + var unknown = CoverageReport.FindUnknownBlockedBy("edge-cases/a", WordsBlockedBy("R1"), new[] { "r1" }); + + Assert.That(unknown, Has.Count.EqualTo(1)); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs new file mode 100644 index 000000000..968353535 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs @@ -0,0 +1,895 @@ +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A fixture flagged fieldworks_producible: true must not use a grammar shape HCLoader never +/// writes. The flag is authored, so these checks can only prove it wrong. Most of the checks come from the +/// census table conformance/fieldworks-writable-shapes.tsv, which says per rule kind which DTD members +/// HCLoader writes; reads it. The checks written here are the ones a +/// table row cannot say: a linear stratum that holds rules or templates, and a partial stem entry +/// with a part of speech. Each class of violation has one test that must fail on a synthetic grammar. +/// +[TestFixture] +public sealed class FieldWorksProducibleShapeTests +{ + private static readonly Lazy> LoadedTable = new(() => + FieldWorksWritableShapes.LoadTable(FieldWorksWritableShapes.TablePath(ConformanceSuite.Root())) + ); + + private static Dictionary<(string Kind, string Member), string> Table => LoadedTable.Value; + + // The 16 stress fixtures sit in the suite under stress/ and obey the same FieldWorks-producible rule. + private static List DiscoverProducible() + { + string root = ConformanceSuite.Root(); + var fixtures = Fixture.DiscoverAll(root); + fixtures.AddRange(Fixture.DiscoverAll(Path.Combine(root, "stress"))); + return fixtures.Where(fixture => fixture.Words.FieldworksProducible == true).ToList(); + } + + [Test] + public void FlaggedFixturesUseNoShapeHcLoaderCannotWrite() + { + List fixtures = DiscoverProducible(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(25)); + + string[] offences = fixtures + .SelectMany(fixture => FindUnproducibleShapes(fixture.GrammarPath).Select(o => $"{fixture.Id}: {o}")) + .OrderBy(line => line, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + offences, + Is.Empty, + "these fixtures are flagged fieldworks_producible but use a shape HCLoader never writes; " + + "rewrite the grammar and re-run the oracle, or delete the fixture:\n " + + string.Join("\n ", offences) + ); + } + + [Test] + public void TheCensusTableNamesOnlyMembersTheDtdDeclares() + { + string dtd = File.ReadAllText(Path.Combine(ConformanceSuite.Root(), "HermitCrabInput.dtd")); + var elements = new HashSet( + Regex.Matches(dtd, @" m.Groups[1].Value), + StringComparer.Ordinal + ); + var attributes = new HashSet(StringComparer.Ordinal); + foreach (Match list in Regex.Matches(dtd, @"", RegexOptions.Singleline)) + { + foreach (Match attribute in Regex.Matches(list.Groups[1].Value, @"^\s*(\w+)\s", RegexOptions.Multiline)) + attributes.Add(attribute.Groups[1].Value); + } + + string[] unknown = Table + .Keys.Select(key => key.Member) + .Distinct(StringComparer.Ordinal) + .Where(member => member != "*") + .Where(member => + member.StartsWith('@') + ? !attributes.Contains(member.TrimStart('@').Split('=')[0]) + : !elements.Contains(member) + ) + .OrderBy(member => member, StringComparer.Ordinal) + .ToArray(); + + Assert.That(unknown, Is.Empty, "the census table names members the DTD does not declare"); + } + + [Test] + public void TheCensusTableGivesALoaderLineToEveryRowThatSaysHcLoaderWritesIt() + { + string[] rows = File.ReadAllLines(FieldWorksWritableShapes.TablePath(ConformanceSuite.Root())) + .Skip(1) + .ToArray(); + string[] missing = rows.Select(row => row.Split('\t')) + .Where(cells => cells[2] is "writes" or "never" or "omitted" && cells[3].Length == 0) + .Select(cells => $"{cells[0]} {cells[1]}") + .OrderBy(row => row, StringComparer.Ordinal) + .ToArray(); + + Assert.That(missing, Is.Empty, "census rows without a HCLoader.cs line"); + } + + [Test] + public void TheCheckFindsAMemberTheCensusTableDoesNotList() + { + string[] found = Scan( + """ + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "derivational-rule mrNew uses @newAttribute, which the census table does not list" }) + ); + } + + [Test] + public void TheCheckFindsARealizationalRule() + { + string[] found = Scan( + """ + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "realizational-rule rrOne is never written by HCLoader (HCLoader.cs 1027)" }) + ); + } + + [Test] + public void TheCheckFindsASlotRuleWithAnOutputPartOfSpeech() + { + string[] found = Scan( + """ + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "slot-rule mrSlot uses @outputPartOfSpeech, which HCLoader never writes" }) + ); + } + + [Test] + public void TheCheckFindsOutputFeaturesOutputMprAndARequiredStemNameOnASlotRule() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "slot-rule mrSlot uses @requiredStemName, which HCLoader never writes", + "slot-rule mrSlot uses OutputHeadFeatures, which HCLoader never writes", + "slot-rule/output sub1 uses @MPRFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsHeadFeatureRequirementsOnACompoundingRule() + { + string[] found = Scan( + """ + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crHead uses HeadRequiredHeadFeatures, which HCLoader never writes", + "compounding-rule crHead uses NonHeadRequiredHeadFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsAnEmptyStemNameRegion() + { + string[] found = Scan( + """ + + """, + """ + + e + + f + + + g + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "stem name snEmpty has an empty region, which HCLoader skips (HCLoader.cs 223)", + "stem name snNoFeatures has an empty region, which HCLoader skips (HCLoader.cs 223)", + } + ) + ); + } + + [Test] + public void TheCheckFindsARequiredEnvironmentOnASubruleShapeHcLoaderGivesNone() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule infix has a RequiredEnvironments on an output of CopyFromInput, InsertSegments, " + + "CopyFromInput; HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix", + "subrule process has a RequiredEnvironments on an output of CopyFromInput, ModifyFromInput; " + + "HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix", + } + ) + ); + } + + [Test] + public void TheCheckFindsRequiredHeadFeaturesOnASubruleShapeHcLoaderGivesNone() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule circumfix is a circumfix with RequiredHeadFeatures; HCLoader writes it only on a lone affix form", + "subrule process is an affix process with RequiredHeadFeatures; HCLoader writes it only on a lone affix form", + } + ) + ); + } + + [Test] + public void TheCheckFindsRequiredHeadFeaturesOnACliticSubrule() + { + string[] found = Scan( + """ + + Clitics + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "clitic-rule/subrule enclitic uses RequiredHeadFeatures, which HCLoader never writes" }) + ); + } + + [Test] + public void TheCheckFindsACapOnAnExocompound() + { + string[] found = Scan( + """ + + + exo + exo + endo + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding rule exoLeft is half of an exocompound, which HCLoader never gives a cap; " + + "it has multipleApplication=\"3\"", + } + ) + ); + } + + [Test] + public void TheCheckFindsMprFeaturesHcLoaderCannotPlace() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + """, + """ + + + + + + inflClasses + + exceptionFeatures + + custom + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "MPR feature group custom is not one of the three groups HCLoader builds", + "MPR feature group inflClasses has matchType=\"all\"; HCLoader builds \"any\"", + "MPR feature group inflClasses has outputType=\"append\", which HCLoader never sets", + "MPR feature mprLoose is in no group; HCLoader puts every MPR feature in one of three groups", + "derivational rule mrDeriv has different MPR lists on its subrules; " + + "HCLoader gives every allomorph the same list (HCLoader.cs 1015-1018)", + "derivational-rule/input subB uses @excludedMPRFeatures, which HCLoader never writes", + "rule mrDeriv excludes MPR feature mprExc; HCLoader excludes only the inflection-type " + + "features of a slot (HCLoader.cs 1766)", + "rule mrSlot excludes MPR feature mprExc; HCLoader excludes only the inflection-type " + + "features of a slot (HCLoader.cs 1766)", + } + ) + ); + } + + [Test] + public void TheCheckAcceptsAnInflectionTypeFeatureExcludedByASlotRule() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """, + """ + + + + lexEntryInflTypes + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckAcceptsAttributesHcLoaderOmitsOrNeverReads() + { + string[] found = Scan( + """ + + + + + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckFindsALinearStratumThatHoldsAMorphologicalRuleOrATemplate() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "stratum #1 is linear (the DTD default) but holds morphological rules mrA", + "stratum #2 is linear but holds morphological rules mrB", + "stratum #3 is linear but holds 1 affix template(s)", + } + ) + ); + } + + [Test] + public void TheCheckAllowsLinearStrataThatHoldNoActiveRuleOrTemplate() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckFindsPartialShapesHcLoaderCannotWrite() + { + string[] found = Scan( + """ + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "partial stem entry eBad has a partOfSpeech", + "slot-rule mrSlot uses @partial=true, which HCLoader never writes", + "unslotted-rule mrLoose uses @outputPartOfSpeech, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsFootFeatures() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + """, + "" + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crFoot uses HeadRequiredFootFeatures, which HCLoader never writes", + "derivational-rule mrFoot uses OutputFootFeatures, which HCLoader never writes", + "derivational-rule mrFoot uses RequiredFootFeatures, which HCLoader never writes", + "lexical-entry eFoot uses AssignedFootFeatures, which HCLoader never writes", + "the language defines FootFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsExcludedEnvironments() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "derivational-rule/subrule subExcl uses ExcludedEnvironments, which HCLoader never writes", + "root-allomorph aExcl uses ExcludedEnvironments, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsARuleThatIsNotBlockable() + { + string[] found = Scan( + """ + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crNo uses @blockable=false, which HCLoader never writes", + "derivational-rule mrNo uses @blockable=false, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsAWrongSideEnvironmentOnALonePrefixOrSuffix() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule prefixRightBad is a lone prefix with a right environment; HCLoader writes only a left one", + "subrule suffixBothBad is a lone suffix with a left environment; HCLoader writes only a right one", + "subrule suffixLeftBad is a lone suffix with a left environment; HCLoader writes only a right one", + } + ) + ); + } + + [Test] + public void TheCheckFindsAMaximumApplicationCountHcLoaderCannotSet() + { + string[] found = Scan( + """ + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crZero uses @multipleApplication=0, which HCLoader never writes", + "derivational-rule mrTwice uses @multipleApplication=2, which HCLoader never writes", + } + ) + ); + } + + private static string[] Scan(string strata, string languageExtras = "") + { + string path = Path.Combine(Path.GetTempPath(), $"hc-shape-{Guid.NewGuid():N}.xml"); + File.WriteAllText( + path, + $"{languageExtras}{strata}" + ); + try + { + return FindUnproducibleShapes(path); + } + finally + { + File.Delete(path); + } + } + + private static readonly string[] RuleElements = ["MorphologicalRule", "RealizationalRule", "CompoundingRule"]; + + private static string[] FindUnproducibleShapes(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + List offences = FieldWorksWritableShapes.Check(doc, Table); + + foreach (XElement entry in doc.Descendants("LexicalEntry")) + { + if ((string?)entry.Attribute("partial") == "true" && entry.Attribute("partOfSpeech") is not null) + offences.Add($"partial stem entry {(string?)entry.Attribute("id")} has a partOfSpeech"); + } + + // HCLoader builds every stratum that holds rules or templates as Unordered, and the DTD default is + // linear. The engine reads the order for the listed rules and when it applies templates. + int number = 0; + foreach (XElement stratum in doc.Descendants("Stratum")) + { + number++; + string? order = (string?)stratum.Attribute("morphologicalRuleOrder"); + if (order == "unordered") + continue; + + var active = new HashSet(StringComparer.Ordinal); + foreach ( + XElement rule in stratum + .Elements("MorphologicalRuleDefinitions") + .Elements() + .Where(e => RuleElements.Contains(e.Name.LocalName)) + ) + { + string? id = (string?)rule.Attribute("id"); + if (id is not null && (string?)rule.Attribute("isActive") != "no") + active.Add(id); + } + + string how = order is null ? "linear (the DTD default)" : "linear"; + string[] held = Split((string?)stratum.Attribute("morphologicalRules")).Where(active.Contains).ToArray(); + if (held.Length > 0) + offences.Add($"stratum #{number} is {how} but holds morphological rules {string.Join(" ", held)}"); + + int templates = stratum + .Elements("AffixTemplates") + .Elements("AffixTemplate") + .Count(template => (string?)template.Attribute("isActive") != "no"); + if (templates > 0) + offences.Add($"stratum #{number} is {how} but holds {templates} affix template(s)"); + } + + return offences.OrderBy(line => line, StringComparer.Ordinal).ToArray(); + } + + private static string[] Split(string? ids) => + string.IsNullOrWhiteSpace(ids) ? [] : ids.Split(' ', StringSplitOptions.RemoveEmptyEntries); +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs new file mode 100644 index 000000000..f2dff0dbf --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs @@ -0,0 +1,432 @@ +using System.Xml.Linq; +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// The check behind conformance/fieldworks-writable-shapes.tsv, the census of what HCLoader can write. +/// Every element and attribute a flagged grammar puts on a rule, template, stratum, stem name, entry, allomorph +/// or phonological rule is looked up there by the kind it sits on. A member the table calls never, or +/// does not list, is an offence. The conditions a table row cannot say (an empty stem-name region, where an +/// environment may sit, which MPR feature group a feature belongs to, a cap on an exocompound) are checked here. +/// +internal static class FieldWorksWritableShapes +{ + private static readonly string[] EnumAttributes = + [ + "isActive", + "partial", + "blockable", + "final", + "optional", + "morphologicalRuleOrder", + "cyclicity", + "phonologicalRuleOrder", + "multipleApplication", + "multipleApplicationOrder", + "isBound", + "redupMorphType", + "type", + "adjacency", + ]; + + // LoadLanguage (HCLoader.cs 181-205) builds exactly these three groups, all with the default output. + private static readonly Dictionary LoaderGroups = new(StringComparer.Ordinal) + { + ["inflClasses"] = "any", + ["exceptionFeatures"] = "all", + ["lexEntryInflTypes"] = "all", + }; + + private static readonly string[] Verdicts = ["writes", "metadata", "never", "omitted", "inert", "shape"]; + + public static string TablePath(string suiteRoot) => Path.Combine(suiteRoot, "fieldworks-writable-shapes.tsv"); + + public static Dictionary<(string Kind, string Member), string> LoadTable(string path) + { + var table = new Dictionary<(string, string), string>(); + string[] lines = File.ReadAllLines(path); + Assert.That(lines[0], Is.EqualTo("kind\tmember\tverdict\tloader\tnote")); + foreach (string line in lines.Skip(1).Where(l => l.Length > 0)) + { + string[] cells = line.Split('\t'); + Assert.That(cells, Has.Length.EqualTo(5), line); + Assert.That(Verdicts, Does.Contain(cells[2]), line); + Assert.That(table.TryAdd((cells[0], cells[1]), cells[2]), Is.True, $"duplicate row: {line}"); + } + return table; + } + + public static IEnumerable<(string Kind, string Member, string Verdict)> Rows( + Dictionary<(string Kind, string Member), string> table + ) => table.Select(pair => (pair.Key.Kind, pair.Key.Member, pair.Value)); + + public static List Check(XDocument doc, Dictionary<(string Kind, string Member), string> table) + { + var offences = new List(); + var slotRuleIds = new HashSet(StringComparer.Ordinal); + foreach (XElement slot in doc.Descendants("Slot")) + slotRuleIds.UnionWith(Split((string?)slot.Attribute("morphologicalRules"))); + + var cliticRuleIds = new HashSet(StringComparer.Ordinal); + foreach (XElement stratum in doc.Descendants("Stratum").Where(s => (string?)s.Element("Name") == "Clitics")) + cliticRuleIds.UnionWith(Split((string?)stratum.Attribute("morphologicalRules"))); + + foreach (XElement element in doc.Descendants()) + { + string? kind = KindOf(element, slotRuleIds, cliticRuleIds); + if (kind is null) + continue; + if (kind == "realizational-rule") + { + offences.Add($"{kind} {Label(element)} is never written by HCLoader (HCLoader.cs 1027)"); + continue; + } + + foreach ((string specific, string generic) in Members(element)) + { + if (!table.TryGetValue((kind, specific), out string? verdict)) + table.TryGetValue((kind, generic), out verdict); + if (verdict is null) + offences.Add($"{kind} {Label(element)} uses {specific}, which the census table does not list"); + else if (verdict == "never") + offences.Add($"{kind} {Label(element)} uses {specific}, which HCLoader never writes"); + } + } + + if (doc.Descendants("FootFeatures").Any()) + offences.Add("the language defines FootFeatures, which HCLoader never writes"); + + CheckStemNameRegions(doc, offences); + CheckEnvironmentShapes(doc, offences); + CheckSubruleHeadFeatureShapes(doc, offences); + CheckExocompoundCaps(doc, offences); + CheckMprFeatureGroups(doc, slotRuleIds, offences); + return offences; + } + + private static IEnumerable<(string Specific, string Generic)> Members(XElement element) + { + foreach (XAttribute attribute in element.Attributes().Where(a => !a.IsNamespaceDeclaration)) + { + string name = attribute.Name.LocalName; + yield return EnumAttributes.Contains(name) + ? ($"@{name}={attribute.Value}", $"@{name}") + : ($"@{name}", $"@{name}"); + } + foreach (XElement child in element.Elements()) + yield return (child.Name.LocalName, child.Name.LocalName); + } + + private static string Label(XElement element) + { + for (XElement? current = element; current is not null; current = current.Parent) + { + string? label = (string?)current.Attribute("id") ?? (string?)current.Element("Name"); + if (label is not null) + return label; + } + return "?"; + } + + private static string? KindOf(XElement element, HashSet slotRuleIds, HashSet cliticRuleIds) + { + switch (element.Name.LocalName) + { + case "MorphologicalRule": + return AffixRuleKind(element, slotRuleIds, cliticRuleIds); + case "CompoundingRule": + return "compounding-rule"; + case "RealizationalRule": + return "realizational-rule"; + case "MorphologicalSubrule": + return AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/subrule"; + case "MorphologicalInput": + return AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/input"; + case "MorphologicalOutput": + return element.Parent?.Name.LocalName == "CompoundingSubrule" + ? "compounding-rule/output" + : AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/output"; + case "CompoundingSubrule": + return "compounding-rule/subrule"; + case "HeadMorphologicalInput": + return "compounding-rule/head-input"; + case "NonHeadMorphologicalInput": + return "compounding-rule/nonhead-input"; + case "AffixTemplate": + return "template"; + case "Slot": + return "slot"; + case "Stratum": + return "stratum"; + case "StemName": + return "stem-name"; + case "Region": + return "stem-region"; + case "LexicalEntry": + return "lexical-entry"; + case "Allomorph": + return "root-allomorph"; + case "PhonologicalRule": + return "phonological-rule"; + case "PhonologicalSubrule": + return "phonological-subrule"; + case "MetathesisRule": + return "metathesis-rule"; + case "MorphemeCoOccurrenceRule": + case "AllomorphCoOccurrenceRule": + return "cooccurrence-rule"; + default: + return null; + } + } + + private static string AffixRuleKindOf(XElement inner, HashSet slotRuleIds, HashSet cliticRuleIds) + { + XElement? rule = inner.Ancestors("MorphologicalRule").FirstOrDefault(); + return rule is null ? "unknown-rule" : AffixRuleKind(rule, slotRuleIds, cliticRuleIds); + } + + // A rule in a slot is inflectional, a rule of the Clitics stratum is a clitic rule, a partial rule outside + // a slot is unslotted, and the rest are derivational (HCLoader.cs 926-1093). + private static string AffixRuleKind(XElement rule, HashSet slotRuleIds, HashSet cliticRuleIds) + { + string id = (string?)rule.Attribute("id") ?? ""; + if (slotRuleIds.Contains(id)) + return "slot-rule"; + if (cliticRuleIds.Contains(id)) + return "clitic-rule"; + return (string?)rule.Attribute("partial") == "true" ? "unslotted-rule" : "derivational-rule"; + } + + // HCLoader keeps a stem name only when it has a non-empty region (HCLoader.cs 223), so an empty region is a + // grammar FieldWorks cannot write: it would leave no stem name at all. + private static void CheckStemNameRegions(XDocument doc, List offences) + { + foreach (XElement region in doc.Descendants("Region")) + { + XElement? head = region.Element("AssignedHeadFeatures"); + if (head is null || !head.Elements("FeatureValue").Any()) + offences.Add($"stem name {Label(region)} has an empty region, which HCLoader skips (HCLoader.cs 223)"); + } + } + + // HCLoader writes an environment on a lone prefix (left), a lone suffix (right) and a circumfix (both). + // An infix, a reduplication and an affix process add none (HCLoader.cs 1371, 1576-1654). + private static void CheckEnvironmentShapes(XDocument doc, List offences) + { + foreach (XElement subrule in doc.Descendants("MorphologicalSubrule")) + { + XElement? required = subrule.Element("RequiredEnvironments"); + if (required is null) + continue; + + string[] output = (subrule.Element("MorphologicalOutput")?.Elements() ?? []) + .Select(e => e.Name.LocalName) + .ToArray(); + bool left = required.Descendants("LeftEnvironment").Any(); + bool right = required.Descendants("RightEnvironment").Any(); + string id = (string?)subrule.Attribute("id") ?? "?"; + if (output.SequenceEqual(["InsertSegments", "CopyFromInput"])) + { + if (right) + { + offences.Add( + $"subrule {id} is a lone prefix with a right environment; HCLoader writes only a left one" + ); + } + } + else if (output.SequenceEqual(["CopyFromInput", "InsertSegments"])) + { + if (left) + { + offences.Add( + $"subrule {id} is a lone suffix with a left environment; HCLoader writes only a right one" + ); + } + } + else if (!output.SequenceEqual(["InsertSegments", "CopyFromInput", "InsertSegments"])) + { + offences.Add( + $"subrule {id} has a RequiredEnvironments on an output of {string.Join(", ", output)}; " + + "HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix" + ); + } + } + } + + // HCLoader sets a subrule's head features only on a lone affix form, not a circumfix (1100-1127) or an affix + // process (ModifyFromInput, 1143, 1458-1466). + private static void CheckSubruleHeadFeatureShapes(XDocument doc, List offences) + { + foreach (XElement subrule in doc.Descendants("MorphologicalSubrule")) + { + if (subrule.Element("RequiredHeadFeatures") is null) + continue; + + string[] output = (subrule.Element("MorphologicalOutput")?.Elements() ?? []) + .Select(e => e.Name.LocalName) + .ToArray(); + string id = (string?)subrule.Attribute("id") ?? "?"; + if (output.SequenceEqual(["InsertSegments", "CopyFromInput", "InsertSegments"])) + { + offences.Add( + $"subrule {id} is a circumfix with RequiredHeadFeatures; HCLoader writes it only on a lone affix form" + ); + } + else if (output.Contains("ModifyFromInput")) + { + offences.Add( + $"subrule {id} is an affix process with RequiredHeadFeatures; HCLoader writes it only on a lone affix form" + ); + } + } + } + + // An exocompound becomes two mirror rules with the same name and no cap (HCLoader.cs 1971-2050). + // Only an endocompound reads the cap, from the parser parameters (1943-1945). + private static void CheckExocompoundCaps(XDocument doc, List offences) + { + XElement[] compounds = doc.Descendants("CompoundingRule").ToArray(); + foreach (XElement rule in compounds) + { + string? cap = (string?)rule.Attribute("multipleApplication"); + if (cap is null or "1") + continue; + if (compounds.Any(other => !ReferenceEquals(other, rule) && IsMirror(rule, other))) + { + offences.Add( + $"compounding rule {Label(rule)} is half of an exocompound, which HCLoader never gives a cap; " + + $"it has multipleApplication=\"{cap}\"" + ); + } + } + } + + private static bool IsMirror(XElement a, XElement b) => + (string?)a.Element("Name") is not null + && (string?)a.Element("Name") == (string?)b.Element("Name") + && ((string?)a.Attribute("headPartsOfSpeech") ?? "") == ((string?)b.Attribute("nonHeadPartsOfSpeech") ?? "") + && ((string?)a.Attribute("nonHeadPartsOfSpeech") ?? "") == ((string?)b.Attribute("headPartsOfSpeech") ?? "") + && (string?)a.Attribute("outputPartOfSpeech") == (string?)b.Attribute("outputPartOfSpeech"); + + // HCLoader puts each MPR feature in inflClasses, exceptionFeatures or lexEntryInflTypes (181-205). + // A slot rule excludes only inflection types (1766); an entry has one inflection class (745-751). + private static void CheckMprFeatureGroups(XDocument doc, HashSet slotRuleIds, List offences) + { + XElement? features = doc.Descendants("MorphologicalPhonologicalRuleFeatures").FirstOrDefault(); + if (features is null) + return; + + var groupOf = new Dictionary(StringComparer.Ordinal); + foreach (XElement group in features.Elements("MorphologicalPhonologicalRuleFeatureGroup")) + { + string name = (string?)group.Element("Name") ?? "?"; + string matchType = (string?)group.Attribute("matchType") ?? "any"; + string outputType = (string?)group.Attribute("outputType") ?? "overwrite"; + if (!LoaderGroups.TryGetValue(name, out string? expected)) + offences.Add($"MPR feature group {name} is not one of the three groups HCLoader builds"); + else if (expected != matchType) + offences.Add($"MPR feature group {name} has matchType=\"{matchType}\"; HCLoader builds \"{expected}\""); + if (outputType != "overwrite") + offences.Add($"MPR feature group {name} has outputType=\"{outputType}\", which HCLoader never sets"); + foreach (string feature in Split((string?)group.Attribute("features"))) + groupOf[feature] = name; + } + + foreach (XElement feature in features.Elements("MorphologicalPhonologicalRuleFeature")) + { + string id = (string?)feature.Attribute("id") ?? "?"; + if (!groupOf.ContainsKey(id)) + { + offences.Add( + $"MPR feature {id} is in no group; HCLoader puts every MPR feature in one of three groups" + ); + } + } + + string GroupOf(string feature) => groupOf.TryGetValue(feature, out string? g) ? g : ""; + + foreach (XElement rule in doc.Descendants("MorphologicalRule")) + { + string id = (string?)rule.Attribute("id") ?? "?"; + bool inSlot = slotRuleIds.Contains(id); + var requiredLists = new HashSet(StringComparer.Ordinal); + var outputLists = new HashSet(StringComparer.Ordinal); + foreach (XElement subrule in rule.Descendants("MorphologicalSubrule")) + { + string[] required = Split( + (string?)subrule.Element("MorphologicalInput")?.Attribute("requiredMPRFeatures") + ); + string[] excluded = Split( + (string?)subrule.Element("MorphologicalInput")?.Attribute("excludedMPRFeatures") + ); + string[] output = Split((string?)subrule.Element("MorphologicalOutput")?.Attribute("MPRFeatures")); + requiredLists.Add(string.Join(" ", required.OrderBy(f => f, StringComparer.Ordinal))); + outputLists.Add(string.Join(" ", output.OrderBy(f => f, StringComparer.Ordinal))); + + foreach (string feature in excluded.Where(f => !inSlot || GroupOf(f) != "lexEntryInflTypes")) + { + offences.Add( + $"rule {id} excludes MPR feature {feature}; HCLoader excludes only the " + + "inflection-type features of a slot (HCLoader.cs 1766)" + ); + } + foreach (string feature in required.Concat(output).Where(f => GroupOf(f) == "lexEntryInflTypes")) + { + offences.Add( + $"rule {id} requires or writes inflection-type MPR feature {feature}; HCLoader never does" + ); + } + if (output.Count(f => GroupOf(f) == "inflClasses") > 1) + offences.Add($"rule {id} writes more than one inflection class; a derivational affix writes one"); + } + + bool derivational = !inSlot && (string?)rule.Attribute("partial") != "true"; + if (derivational && (requiredLists.Count > 1 || outputLists.Count > 1)) + { + offences.Add( + $"derivational rule {id} has different MPR lists on its subrules; " + + "HCLoader gives every allomorph the same list (HCLoader.cs 1015-1018)" + ); + } + } + + foreach (XElement entry in doc.Descendants("LexicalEntry")) + { + if (Split((string?)entry.Attribute("ruleFeatures")).Count(f => GroupOf(f) == "inflClasses") > 1) + offences.Add($"entry {Label(entry)} has more than one inflection class"); + } + + foreach (XElement rule in doc.Descendants("CompoundingRule")) + { + foreach ( + XAttribute attribute in rule.Attributes() + .Where(a => a.Name.LocalName.EndsWith("ProdRestrictionsMprFeatures", StringComparison.Ordinal)) + ) + { + foreach (string feature in Split(attribute.Value).Where(f => GroupOf(f) != "exceptionFeatures")) + { + offences.Add( + $"compounding rule {Label(rule)} lists {feature} as a production restriction, " + + "which is not an exceptionFeatures feature" + ); + } + } + foreach (XElement output in rule.Descendants("MorphologicalOutput")) + { + string[] written = Split((string?)output.Attribute("MPRFeatures")); + if (written.Length > 1 || written.Any(f => GroupOf(f) != "inflClasses")) + { + offences.Add( + $"compounding rule {Label(rule)} writes {string.Join(" ", written)}; " + + "HCLoader writes one inflection class (HCLoader.cs 1950)" + ); + } + } + } + } + + private static string[] Split(string? ids) => + string.IsNullOrWhiteSpace(ids) ? [] : ids.Split(' ', StringSplitOptions.RemoveEmptyEntries); +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs new file mode 100644 index 000000000..4a9547cae --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs @@ -0,0 +1,112 @@ +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class GuesserSignatureTests +{ + private const string GrammarXml = """ + + + + + GuesserSignatureProbe + + n + + + Main + + f + o + s + + + + Any + + + + Main + + + plural + + + + s + + + PL + + + + + + [Any]* + + + + + + + """; + + private static string WriteTempGrammar() + { + string path = Path.Combine(Path.GetTempPath(), "hc-guesser-signature-" + Guid.NewGuid().ToString("N") + ".xml"); + File.WriteAllText(path, GrammarXml); + return path; + } + + // Named for the overload it calls, not the production path: no Tool-project reference means + // it cannot call ParseOneWord. Proves the fact that path relies on: the 2-argument overload + // never guesses. + [Test] + public void TwoArgumentParseWord_NeverGuesses_ZeroResultsForAnUnknownRoot() + { + string path = WriteTempGrammar(); + try + { + Language language = XmlLanguageLoader.Load(path); + var morpher = new Morpher(new TraceManager(), language); + + Word[] results = morpher.ParseWord("foos", out _).ToArray(); + + Assert.That(results, Is.Empty); + Assert.That(SignatureFormat.BuildSignature(results), Is.EqualTo("-")); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void GuessRootTrue_RendersGuessedStemAsItsOwnLiteralSurfaceText() + { + string path = WriteTempGrammar(); + try + { + Language language = XmlLanguageLoader.Load(path); + var morpher = new Morpher(new TraceManager(), language); + + Word[] results = morpher.ParseWord("foos", out _, guessRoot: true).ToArray(); + + Assert.That(results, Is.Not.Empty); + string signature = SignatureFormat.BuildSignature(results); + + Assert.That(signature, Is.EqualTo("foo+PL|foos;foos|foos")); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs new file mode 100644 index 000000000..96d56da1b --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs @@ -0,0 +1,136 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A harness-level problem is never a crash match, expectCrash or not. A misconfigured adapter +/// (wrong path, an argument it rejects, ...) exits nonzero on every fixture it is asked to run, not +/// just the one pinning a crash -- these tests use a stub (no real subprocess) +/// to prove tells the two apart. +/// +[TestFixture] +public sealed class MaterializedRunnerCrashClassificationTests +{ + private sealed class StubEngine(params string[] crashingFixtureIds) : IEngine + { + private readonly HashSet _crashingFixtureIds = new(crashingFixtureIds, StringComparer.Ordinal); + + public string Name => "stub"; + public IReadOnlySet Capabilities { get; } = new HashSet(); + + public List Run(MaterializedFixture fixture) + { + if (_crashingFixtureIds.Contains(fixture.Id)) + throw new EngineCrashException($"stub crash for '{fixture.Id}'"); + return new List(); + } + } + + private static MaterializedFixture CreateFixture( + string root, + string id, + bool expectCrash, + bool requiresPhonology = false, + bool pathological = false + ) + { + string directory = Path.Combine(root, id); + Directory.CreateDirectory(directory); + // A PhonologicalRule is what RequiresDerivation.Derive reads as "phonology"; without one the + // manifest's empty Requires matches the grammar. + File.WriteAllText( + Path.Combine(directory, "grammar.xml"), + requiresPhonology ? "" : "" + ); + File.WriteAllText(Path.Combine(directory, "words.txt"), "w\n"); + File.WriteAllText(Path.Combine(directory, "expected.tsv"), ""); + var manifest = new FixtureManifest { Id = id, ExpectCrash = expectCrash }; + if (requiresPhonology) + manifest.Requires.Add("phonology"); + if (pathological) + manifest.Category = "pathological"; + return new MaterializedFixture(directory, manifest); + } + + private static MaterializedRunReport Run(string root, IEngine engine) + { + var fixtures = new List + { + CreateFixture(root, "crash-fixture", expectCrash: true), + CreateFixture(root, "control-fixture", expectCrash: false), + }; + return MaterializedRunner.Run(fixtures, engine, includePathological: false); + } + + [Test] + public void ExpectCrashPassesWhenTheSameEngineStillCompletesAnOrdinaryFixture() + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Genuine crash: the stub only fails on the fixture that pins one. + MaterializedRunReport report = Run(root, new StubEngine("crash-fixture")); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Passed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public void ExpectCrashDoesNotPassWhenTheEngineFailsEveryFixtureNotJustTheCrashOne() + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Misconfigured adapter: rejects its own arguments and exits nonzero regardless of which + // fixture it was asked to run. Must not be credited as reproducing the pinned crash. + MaterializedRunReport report = Run(root, new StubEngine("crash-fixture", "control-fixture")); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + Assert.That(result.Reason, Does.Contain("control-fixture")); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [TestCase(true, false, TestName = "ExpectCrashSkipsAControlTheEngineLacksACapabilityFor")] + [TestCase(false, true, TestName = "ExpectCrashSkipsAControlExcludedAsPathological")] + public void ExpectCrashUsesOnlyAControlTheRunWouldExecute(bool requiresPhonology, bool pathological) + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // The stub declares no capabilities and fails the ineligible fixture, as an engine that + // cannot run it would. Only the eligible control says whether the adapter works. + var fixtures = new List + { + CreateFixture(root, "crash-fixture", expectCrash: true), + CreateFixture(root, "ineligible-control", expectCrash: false, requiresPhonology, pathological), + CreateFixture(root, "control-fixture", expectCrash: false), + }; + MaterializedRunReport report = MaterializedRunner.Run( + fixtures, + new StubEngine("crash-fixture", "ineligible-control"), + includePathological: false + ); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Passed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs new file mode 100644 index 000000000..599cd0a47 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs @@ -0,0 +1,94 @@ +using System.Text.RegularExpressions; +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A child process spawned while Xplat Code Coverage is attached inherits its CLR-profiler +/// environment variables unless something strips them first; a profiled child that then crashes or +/// is killed has been observed to destabilize the coverage channel the parent test host shares, +/// producing a trace-less test host crash. ChildProcessEnvironment.CreateStartInfo is the one +/// place that strips them, so this gate fails, naming the file, if any other source file under the +/// conformance library or its test project constructs a ProcessStartInfo directly. +/// +[TestFixture] +public sealed class ProcessStartInfoConstructionGateTests +{ + // Tolerates a namespace-qualified spelling of the type name too; a plain substring search on the + // unqualified name missed that spelling on first falsification. + private static readonly Regex RawConstruction = new( + @"new\s+(?:[\w.]+\.)?ProcessStartInfo\b", + RegexOptions.Compiled + ); + private const string FactoryFileName = "ChildProcessEnvironment.cs"; + + // This file's own doc comment and assertion messages necessarily describe the pattern in prose, + // which the regex above would otherwise flag as a violation of itself. + private const string SelfFileName = nameof(ProcessStartInfoConstructionGateTests) + ".cs"; + + private static string RepositoryRoot() + { + string? directory = TestContext.CurrentContext.TestDirectory; + while (directory is not null) + { + // The directory holding Machine.sln; the walk reads only source files, so it needs no suite. + if (File.Exists(Path.Combine(directory, "Machine.sln"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root."); + return string.Empty; + } + + private static IReadOnlyList SourceFilesUnder(string relativeRoot) + { + string root = Path.Combine(RepositoryRoot(), relativeRoot); + Assert.That(Directory.Exists(root), Is.True, $"expected a source directory at '{root}'"); + return Directory + .GetFiles(root, "*.cs", SearchOption.AllDirectories) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}")) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}")) + .ToArray(); + } + + [Test] + public void OnlyChildProcessEnvironmentConstructsAProcessStartInfo() + { + IReadOnlyList files = SourceFilesUnder( + Path.Combine("src", "SIL.Machine.Morphology.HermitCrab.Conformance") + ) + .Concat(SourceFilesUnder(Path.Combine("tests", "SIL.Machine.Morphology.HermitCrab.Tests"))) + .ToArray(); + + // A wrong root or a typo'd search pattern would make every assertion below pass vacuously by + // finding nothing to complain about, so first prove the walk actually reached real content. + Assert.That(files.Count, Is.GreaterThan(50), "the source walk found suspiciously few .cs files"); + string[] factoryFiles = files.Where(path => Path.GetFileName(path) == FactoryFileName).ToArray(); + Assert.That(factoryFiles, Has.Length.EqualTo(1), $"expected exactly one {FactoryFileName} on the walk"); + Assert.That( + RawConstruction.IsMatch(File.ReadAllText(factoryFiles[0])), + Is.True, + $"{FactoryFileName} itself no longer matches the construction pattern this gate looks for" + ); + + List violations = new(); + foreach (string file in files) + { + string name = Path.GetFileName(file); + if (name == FactoryFileName || name == SelfFileName) + continue; + if (RawConstruction.IsMatch(File.ReadAllText(file))) + violations.Add(file); + } + + Assert.That( + violations, + Is.Empty, + "these files construct a ProcessStartInfo directly instead of through " + + "ChildProcessEnvironment.CreateStartInfo, so a coverage-collector profiler attachment " + + $"is not stripped before the child spawns:{Environment.NewLine}" + + string.Join(Environment.NewLine, violations) + ); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs new file mode 100644 index 000000000..485182df4 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs @@ -0,0 +1,37 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ProposePatchWriterTests +{ + [Test] + public void ProposeForAWordTheOracleSkipsSuggestsExpectSkip() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/loader-pattern-shapes"); + // "bit" is a declared expect_skip word. Declaring a parse for it instead makes self-check + // see a mismatch, and the oracle still skips it. + WordEntry word = fixture.Words.Words.Single(w => w.Word == "bit"); + word.ExpectSkip = false; + word.Parses.Add(new ParseEntry { Signature = "deliberately-wrong" }); + + var proposal = new StringWriter(); + Runner.RunSelfCheck( + new List { fixture }, + includePathological: true, + new HashSet { "phonology", MorpherSettings.Capability }, + propose: true, + proposal + ); + + Assert.Multiple(() => + { + Assert.That(proposal.ToString(), Does.Contain("word 'bit' signature mismatch")); + Assert.That(proposal.ToString(), Does.Contain("expect_skip: true")); + Assert.That(proposal.ToString(), Does.Not.Contain("expect_fail: true")); + }); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj index c131a8b81..822610d31 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj @@ -36,6 +36,7 @@ + diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs new file mode 100644 index 000000000..4af6472b3 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs @@ -0,0 +1,104 @@ +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class SuitePinCheckTests +{ + private string _root = ""; + private string _suite = ""; + private string _head = ""; + private string? _override; + + [SetUp] + public void SetUp() + { + _override = Environment.GetEnvironmentVariable(ConformanceSuite.OverrideVariable); + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, null); + _root = Path.Combine(Path.GetTempPath(), $"pin-check-{Guid.NewGuid():N}"); + _suite = Path.Combine(_root, "conformance"); + Directory.CreateDirectory(Path.Combine(_suite, "stress")); + File.WriteAllText(Path.Combine(_suite, "constructs.txt"), "a\n"); + Git(_suite, "init", "-q"); + Git(_suite, "add", "."); + Git( + _suite, + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "x" + ); + _head = Git(_suite, "rev-parse", "HEAD").Trim(); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, _override); + foreach (string path in Directory.EnumerateFileSystemEntries(_root, "*", SearchOption.AllDirectories)) + File.SetAttributes(path, FileAttributes.Normal); + Directory.Delete(_root, recursive: true); + } + + [Test] + public void Warnings_ForAnUntouchedCloneAtThePin_AreEmpty() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), _head + "\n"); + + Assert.That(SuitePinCheck.Warnings(_suite), Is.Empty); + Assert.That(SuitePinCheck.Warnings(Path.Combine(_suite, "stress")), Is.Empty); + } + + [Test] + public void Warnings_ForACloneAtAnotherCommit_SayTheSuiteIsStale() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), new string('a', 40) + "\n"); + + Assert.That(SuitePinCheck.Warnings(_suite), Has.Exactly(1).Contain("stale suite")); + } + + [Test] + public void Warnings_ForAnUntrackedFixtureAtThePin_NameIt() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), _head + "\n"); + File.WriteAllText(Path.Combine(_suite, "extra.txt"), "x"); + + Assert.That(SuitePinCheck.Warnings(_suite), Has.Exactly(1).Contain("extra.txt")); + } + + [Test] + public void Warnings_WhenTheOverrideIsSetOrTheFixturesAreElsewhere_AreEmpty() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), new string('a', 40) + "\n"); + string elsewhere = Path.Combine(_root, "other"); + Directory.CreateDirectory(elsewhere); + Assert.That(SuitePinCheck.Warnings(elsewhere), Is.Empty); + + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, _suite); + Assert.That(SuitePinCheck.Warnings(_suite), Is.Empty); + } + + private static string Git(string workingDirectory, params string[] arguments) + { + ProcessStartInfo startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = workingDirectory; + startInfo.RedirectStandardOutput = true; + foreach (string variable in new[] { "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE" }) + startInfo.EnvironmentVariables.Remove(variable); + foreach (string argument in arguments) + startInfo.ArgumentList.Add(argument); + using Process process = Process.Start(startInfo)!; + string output = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + Assert.That(process.ExitCode, Is.EqualTo(0), $"git {string.Join(' ', arguments)}"); + return output; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs new file mode 100644 index 000000000..05de67bd7 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs @@ -0,0 +1,50 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ToolScriptExitCodeTests +{ + [Test] + public void Main_WithAScriptWhoseFirstCommandFails_ExitsWithThatFailureEvenWhenALaterCommandSucceeds() + { + (int failing, int passing) = RunScripts(["batch --start=x {words} {out}", "batch {words} {out}"]); + + Assert.Multiple(() => + { + Assert.That(failing, Is.Not.EqualTo(0)); + Assert.That(passing, Is.EqualTo(0)); + }); + } + + private static (int Failing, int Passing) RunScripts(string[] failingScript) + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + string directory = Path.Combine(Path.GetTempPath(), $"tool-script-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string words = Path.Combine(directory, "words.txt"); + File.WriteAllLines(words, ["a"]); + string output = Path.Combine(directory, "out.tsv"); + TextWriter originalOut = Console.Out; + try + { + Console.SetOut(TextWriter.Null); + int Run(string[] lines) + { + string script = Path.Combine(directory, $"script-{Guid.NewGuid():N}.txt"); + File.WriteAllLines( + script, + lines.Select(line => line.Replace("{words}", words).Replace("{out}", output)) + ); + return Program.Main(["-i", fixture.GrammarPath, "-s", script]); + } + return (Run(failingScript), Run(["batch {words} {out}"])); + } + finally + { + Console.SetOut(originalOut); + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs new file mode 100644 index 000000000..36fc723e8 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs @@ -0,0 +1,196 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class WordsYamlMorpherSettingsTests +{ + private const string Words = + "words:\n - word: kat\n note: n\n parses:\n - signature: \"KAT|kat\"\n rules: []\n"; + + private static WordsYaml Load(string frontMatter) + { + string path = Path.Combine(Path.GetTempPath(), "words-" + Guid.NewGuid().ToString("N") + ".yaml"); + if ( + !frontMatter.Contains("requires:", StringComparison.Ordinal) + && ( + frontMatter.Contains("morpher:", StringComparison.Ordinal) + || frontMatter.Contains("variants:", StringComparison.Ordinal) + ) + ) + { + frontMatter = "requires: [morpher-settings]\n" + frontMatter; + } + File.WriteAllText(path, "language: T\n" + frontMatter + Words); + try + { + return WordsYamlLoader.Load(path); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void AbsentBlockMeansTheDefaults() + { + WordsYaml words = Load(""); + Assert.That(words.Morpher.IsEmpty, Is.True); + } + + [Test] + public void EveryAcceptedKeyIsRead() + { + WordsYaml words = Load( + "morpher:\n DelReapps: 1\n MaxRoots: 3\n MergeAnalyses: false\n MakePartialsFinal: true\n GuessRoots: false\n" + ); + Assert.Multiple(() => + { + Assert.That(words.Morpher.DelReapps, Is.EqualTo(1)); + Assert.That(words.Morpher.MaxRoots, Is.EqualTo(3)); + Assert.That(words.Morpher.MergeAnalyses, Is.False); + Assert.That(words.Morpher.MakePartialsFinal, Is.True); + Assert.That(words.Morpher.GuessRoots, Is.False); + Assert.That(words.Morpher.GuessRoots, Is.EqualTo(false)); + }); + } + + [TestCase("morpher:\n MaxAlternatives: 5\n", "unknown key 'MaxAlternatives'")] + [TestCase("morpher:\n DelReapps: one\n", "DelReapps")] + [TestCase("morpher:\n DelReapps: -1\n", "DelReapps")] + [TestCase("morpher:\n DelReapps: 2147483648\n", "DelReapps")] + [TestCase("morpher:\n MaxRoots: 0\n", "MaxRoots")] + [TestCase("morpher:\n MaxRoots: 2147483648\n", "MaxRoots")] + [TestCase("morpher:\n MergeAnalyses: yes\n", "expected 'true' or 'false'")] + [TestCase("morpher: {}\n", "empty block")] + [TestCase("morpher: 1\n", "expected a mapping")] + public void AnUnappliableBlockIsAnError(string frontMatter, string expected) + { + var ex = Assert.Throws(() => Load(frontMatter)); + Assert.That(ex!.Message, Does.Contain(expected)); + } + + [Test] + public void TheFactoryAppliesEveryDeclaredValueAndOnlyThose() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + var settings = new MorpherSettings { DelReapps = 2, MakePartialsFinal = true }; + + Morpher morpher = ConformanceMorpherFactory.Create(language, settings); + + Assert.Multiple(() => + { + Assert.That(morpher.DeletionReapplications, Is.EqualTo(2)); + Assert.That(morpher.AlwaysEnforceFinalTemplates, Is.True); + Assert.That(morpher.MaxStemCount, Is.EqualTo(2), "undeclared: constructor default"); + Assert.That(morpher.MergeEquivalentAnalyses, Is.True, "undeclared: constructor default"); + }); + } + + [Test] + public void VariantsAreFullRunsAndRequireTheSameOrderedWords() + { + WordsYaml words = Load( + "morpher:\n DelReapps: 1\nvariants:\n - name: roots-two\n morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n" + ); + Assert.Multiple(() => + { + Assert.That(words.Runs.Select(run => run.Name), Is.EqualTo(new[] { "default", "roots-two" })); + Assert.That(words.Variants.Single().Morpher.MaxRoots, Is.EqualTo(2)); + Assert.That( + words.Variants.Single().Morpher.DelReapps, + Is.Null, + "variant settings start from engine defaults" + ); + Assert.That(words.Variants.Single().Words.Single().Note, Is.EqualTo("n")); + }); + } + + [TestCase("default", "default is reserved")] + [TestCase("BadName", "must match")] + public void InvalidVariantNamesAreRejected(string name, string expected) + { + string yaml = + $"requires: [morpher-settings]\nvariants:\n - name: {name}\n morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n"; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain(expected)); + } + + [Test] + public void DuplicateVariantNamesAreRejected() + { + const string VariantBody = + " morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n"; + string yaml = + "requires: [morpher-settings]\nvariants:\n - name: same\n" + + VariantBody + + " - name: same\n" + + VariantBody; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain("duplicate variant name")); + } + + [Test] + public void VariantWordOrderMustMatchDefault() + { + string yaml = + "requires: [morpher-settings]\nvariants:\n - name: changed\n morpher:\n MaxRoots: 2\n words:\n - word: other\n note: n\n parses:\n - signature: OTHER|other\n rules: []\n"; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain("same order")); + } + + [Test] + public void SelfCheckRunsVariantsAndFailsAWrongVariantExpectationByName() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/deletion-reapplications"); + WordRun variant = fixture.Words.Variants.Single(v => v.Name == "del-reapps-two"); + variant.Words[0].Parses[0].Signature = "deliberately-wrong"; + + RunReport report = Runner.RunSelfCheck( + new List { fixture }, + includePathological: true, + new HashSet { "phonology", MorpherSettings.Capability }, + propose: false, + TextWriter.Null + ); + + FixtureResult failed = report.Results.Single(r => + r.FixtureId.Contains("variant del-reapps-two", StringComparison.Ordinal) + ); + Assert.That(failed.Outcome, Is.EqualTo(FixtureOutcome.Failed)); + Assert.That(failed.Reason, Does.Contain("mismatched")); + } + + [Test] + public void ABlockIsACapabilityRequirement() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/deletion-reapplications"); + + Assert.That( + RequiresDerivation.Derive(fixture.GrammarPath, fixture.Words.Morpher), + Is.EqualTo(new[] { "phonology", MorpherSettings.Capability }) + ); + Assert.That( + RequiresDerivation.Derive(fixture.GrammarPath, MorpherSettings.Defaults), + Is.EqualTo(new[] { "phonology" }) + ); + Assert.That( + RequiresDerivation.Derive( + fixture.GrammarPath, + new[] + { + MorpherSettings.Defaults, + new MorpherSettings { MaxRoots = 2 }, + } + ), + Is.EqualTo(new[] { "phonology", MorpherSettings.Capability }) + ); + } +} From 7952575854aa7071e6323bf44bd4ef44d7532eb1 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 14:19:05 -0500 Subject: [PATCH 04/15] Add an Explicit test that runs the suite's stress fixtures Selected with TestCategory=Stress, StressFixtureParityTests runs the 16 fixtures under conformance/stress through the same comparison as the runner. It never runs by default. Co-Authored-By: Claude Sonnet 5.5 --- .../StressFixtureParityTests.cs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs new file mode 100644 index 000000000..9e420e8e3 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs @@ -0,0 +1,30 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +[Explicit("Timing and optimization fixtures; run only with --filter TestCategory=Stress")] +[Category("Stress")] +public sealed class StressFixtureParityTests +{ + private static string StressRoot() => Path.Combine(ConformanceSuite.Root(), "stress"); + + private static IEnumerable FixtureIds() => Fixture.DiscoverAll(StressRoot()).Select(fixture => fixture.Id); + + [TestCaseSource(nameof(FixtureIds))] + public void ParsesMatchTheMasterEngine(string fixtureId) + { + List fixtures = Fixture.DiscoverAll(StressRoot()).Where(f => f.Id == fixtureId).ToList(); + RunReport report = Runner.RunSelfCheck( + fixtures, + includePathological: true, + engineCapabilities: Runner.DefaultCapabilities, + propose: false, + proposeOutput: TextWriter.Null + ); + string reasons = string.Join("; ", report.Results.Select(r => r.Reason)); + Assert.That(report.Failed, Is.EqualTo(0), reasons); + Assert.That(report.Passed, Is.EqualTo(1), reasons); + } +} From 82cfb375ef813c9261de827da50bc8a674704c4e Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 14:19:08 -0500 Subject: [PATCH 05/15] Run the conformance suite in CI The Conformance workflow builds the runner once in Release, then runs the self-check, the stress self-check, the manifest check and the coverage report, and fails unless the commit in conformance.commit is an ancestor of the suite's main. local_check.sh runs the same checks and honours HC_CONFORMANCE_SUITE. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/conformance.yml | 84 +++++++++++++++++++++++++++++++ local_check.sh | 21 ++++++++ 2 files changed, 105 insertions(+) create mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..aec98da25 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,84 @@ +name: Conformance + +on: + push: + branches: + - "**" + paths: + - "conformance.commit" + - "src/SIL.Machine/**" + - "src/SIL.Machine.Morphology.HermitCrab*/**" + - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**" + - ".github/workflows/conformance.yml" + pull_request: + paths: + - "conformance.commit" + - "src/SIL.Machine/**" + - "src/SIL.Machine.Morphology.HermitCrab*/**" + - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**" + - ".github/workflows/conformance.yml" + +jobs: + conformance: + name: Conformance suite + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 10.0.x + + # Building the runner clones sillsdev/fieldworks-parser-conformance into conformance/ at the commit in + # conformance.commit; see EnsureConformanceSuite.targets. + # + # Release, because edge-cases/deep-optional-affix-nesting has a 15 s wall-clock budget that Debug uses + # most of on a small runner. Built once; every step below runs the built runner. + - name: Build the runner + run: dotnet build -c Release src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj + + # Every fixture's words.yaml must match what the C# engine produces, in process, including the fixtures + # with a budget_ms. + - name: Run conformance self-check + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --include-pathological + + # The suite's stress fixtures (conformance/stress) must agree with the engine too. + - name: Run stress self-check + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance/stress --include-pathological + + # The suite's generated manifest must match a fresh computation, and every grammar and words.yaml must + # validate against the suite's DTD and schema. + - name: Check the suite manifest + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --check-manifest + + # Exits 1 on an uncovered construct, on a dead rule (exercised by zero words) missing from + # conformance/dead-rules-allowed.tsv, on an allowed rule some word now exercises, and when the suite's + # coverage.csv or rules.csv is stale. It writes nothing; the suite owns both files. + - name: Run coverage report + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --coverage-report + + # A pin that is not on the suite's main could be lost when that branch is deleted. + - name: Check the pinned commit is on the suite's main + if: always() + run: | + pin="$(tr -d '[:space:]' < conformance.commit)" + git -C conformance fetch --quiet origin main + if ! git -C conformance merge-base --is-ancestor "$pin" origin/main; then + echo "::error file=conformance.commit::$pin is not an ancestor of origin/main in the conformance suite" + exit 1 + fi diff --git a/local_check.sh b/local_check.sh index e80ed7ff3..b46c37342 100755 --- a/local_check.sh +++ b/local_check.sh @@ -30,12 +30,33 @@ fi # output read through tail cannot pass off a build failure as a test pass. failed=() +# conformance/ is the suite clone at the commit in conformance.commit; the build below clones it or moves it to the +# pin. HC_CONFORMANCE_SUITE= uses another copy. +suite="${HC_CONFORMANCE_SUITE:-conformance}" + dotnet tool restore || failed+=(tool-restore) dotnet restore || failed+=(restore) dotnet csharpier check . || failed+=(csharpier) dotnet build --no-restore -c Release || failed+=(build) dotnet test --verbosity normal || failed+=(test) +# The same checks as .github/workflows/conformance.yml. +hc_conformance=(dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --) +"${hc_conformance[@]}" --fixtures "$suite" --include-pathological || failed+=(conformance) +"${hc_conformance[@]}" --fixtures "$suite/stress" --include-pathological || failed+=(conformance-stress) +"${hc_conformance[@]}" --fixtures "$suite" --check-manifest || failed+=(manifest) +"${hc_conformance[@]}" --fixtures "$suite" --coverage-report || failed+=(coverage-report) +# Nothing above may have changed the suite clone. +git -C "$suite" diff --exit-code || failed+=(suite-modified) +# The pin must be on the suite's main (CI checks the same). Not checked without a network or with another suite. +if [ -z "${HC_CONFORMANCE_SUITE:-}" ]; then + if git -C conformance fetch --quiet origin main; then + git -C conformance merge-base --is-ancestor "$(tr -d '[:space:]' < conformance.commit)" origin/main || failed+=(pin-not-on-main) + else + echo "local_check: could not fetch the suite's main, so the pin was not checked against it" + fi +fi + if [ "${#failed[@]}" -ne 0 ]; then echo "local_check: FAILED (${failed[*]})" exit 1 From dd7e2d2fec2c7608d87cc75f50ddfe4894cd8dc9 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 14:19:14 -0500 Subject: [PATCH 06/15] Document how machine runs the conformance suite The README and AGENTS.md point to the suite repository. A README beside the runner says how the suite is fetched, how to bump the pin, how to regenerate the files the suite owns, how to run another engine with --adapter, and what to run before pushing. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 8 ++ README.md | 2 + .../README.md | 106 ++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md diff --git a/AGENTS.md b/AGENTS.md index 6e2e7194d..1127b8480 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,3 +78,11 @@ first row wins: `docs/review/devils-advocate.md` is an optional adversarial pass for a high-risk change. Add a nested `AGENTS.md` only when a subtree needs different rules. + +`conformance/` is an ignored clone of sillsdev/fieldworks-parser-conformance at +the commit in `conformance.commit`; do not edit it from machine. Building the +HermitCrab tests fetches it and moves it to the pinned commit after a pull or +branch switch. Changes to the grammars land in that repository, following its +`AGENTS.md`. To try an unmerged suite change, set `HC_CONFORMANCE_SUITE=`. +How to get, bump and check the suite from machine is in +`src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md`. diff --git a/README.md b/README.md index 2ad4023a9..1113d822c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Machine provides implementations of many common statistical word alignment model Machine contains a rule-based morphological/phonological parser called HermitCrab. +The HermitCrab conformance grammars are in [sillsdev/fieldworks-parser-conformance](https://github.com/sillsdev/fieldworks-parser-conformance), an engine-neutral suite of grammars a FieldWorks project can produce, each with the parses an engine must return. Machine runs them from a clone in `conformance/`, at the commit named in `conformance.commit`, which the build fetches (`dotnet build` or `dotnet test` of the HermitCrab tests), with the runner in [`src/SIL.Machine.Morphology.HermitCrab.Conformance`](src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md). + ### Feature Structures Machine provides a flexible implementation of feature structures with efficient unification, subsumption, and priority union operations. Feature values can be atomic symbols, strings, or variables. diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md new file mode 100644 index 000000000..62f309a00 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md @@ -0,0 +1,106 @@ +# HermitCrab conformance runner + +`hc-conformance` runs the grammars of [sillsdev/fieldworks-parser-conformance](https://github.com/sillsdev/fieldworks-parser-conformance) +against machine's HermitCrab engine, in process. The suite is a clone in `conformance/` at the root of this repository, +at the one commit named in `conformance.commit`. Commands run from the repository root. `` below means +`dotnet run --project src/SIL.Machine.Morphology.HermitCrab.Conformance -c Release --`. + +## What runs where + +| Check | Where it runs | +|---|---| +| ` --fixtures conformance --include-pathological`, `--fixtures conformance/stress --include-pathological`, `--check-manifest`, `--coverage-report`, and that `conformance.commit` is on the suite's `main` | `.github/workflows/conformance.yml` and `./local_check.sh` | +| every fixture, the manifest, the DTD and schemas, the FieldWorks-producible shape rules | `dotnet test tests/SIL.Machine.Morphology.HermitCrab.Tests` (`ConformanceFixtureGateTests`, `ConformanceManifestTests`, `FieldWorksProducibleShapeTests`) | +| the 16 stress fixtures under NUnit | `dotnet test ... --filter "TestCategory=Stress"`, which is `[Explicit]` and never runs by default | + +## Getting the suite + +Nothing to do: building this project (which `dotnet build` and `dotnet test` of the HermitCrab tests do) runs +`EnsureConformanceSuite.targets`, which makes `conformance/` a clone of the suite at the commit in `conformance.commit`. + +- Absent: it is cloned (`git clone --filter=blob:none`, so the suite's history is available) into a temporary directory + beside `conformance/` and renamed, so an interrupted or parallel build never leaves a half-populated suite. +- At the pinned commit: nothing is fetched and nothing is written, so an offline build works. +- At another commit, for example after `git pull` changed the pin: moved to the pinned commit. If `conformance/` has + uncommitted changes the build stops and names the file; it never touches them. +- At the pinned commit with local changes (edited files, extra fixtures): the build warns and names them, and goes on, + so a suite author can try a change. The runner prints the same warning, and also warns when `conformance/` is not at + the pin, which `dotnet run --no-build` would otherwise read silently. +- No git, no network, a connection that stalls (git gives up after 20 seconds without data, the build after three + minutes in all), or any other failure: the build warns and never fails. The tests then fail with the two commands to + run by hand, `git clone --filter=blob:none https://github.com/sillsdev/fieldworks-parser-conformance.git conformance` + and `git -C conformance checkout --detach `. +- `SIL.Machine`, the HermitCrab library and the HermitCrab tool do not reference this project, so building or packing + them never touches `conformance/`. + +To run against a copy of the suite you are changing, set `HC_CONFORMANCE_SUITE=`. The tests then use +that directory, and the build neither clones nor checks the pin. The runner takes the same path with `--fixtures`. This +is how a change to the suite is tried here before it is merged there. + +Never edit, commit from or write into `conformance/` from machine. A change to a grammar, a `words.yaml`, the manifest, +`coverage.csv` or `rules.csv` is a pull request to the suite repository (see its `AGENTS.md`). No runner flag writes into +the suite. + +## Bumping the pin + +Do this when the suite has a change machine should run, and for no other reason. + +1. The change is merged on the suite's `main` and its CI is green. A pin must be on `main`: CI fails a pin that is not an + ancestor of the suite's `origin/main`. +2. Put the full 40-character commit SHA, nothing else, on the one line of `conformance.commit`. +3. Build once (`dotnet build tests/SIL.Machine.Morphology.HermitCrab.Tests`) so `conformance/` moves to it, then run the + checks under "Before you push". +4. Commit `conformance.commit` with a message that says what is now true, for example "Run the conformance suite at + ``, which adds the `` fixture". +5. If `--check-manifest` or `--coverage-report` reports one of the suite's own files as stale, that is a defect in the + suite. Fix it there and bump to that commit. Do not patch it in `conformance/`. + +## Regenerating what the suite owns + +The suite checks in files that are computed from its fixtures. The runner only checks them. + +- The manifest: in a clone of the suite, `python3 scripts/generate_manifest.py`. Machine's `--check-manifest` must agree + with it byte for byte. +- `coverage.csv` and `rules.csv`: ` --fixtures conformance --coverage-report --coverage-output ` + writes fresh copies into the clone. The output directory may not be inside `--fixtures`. +- The expected rows for a new or changed fixture: ` --fixtures --include-pathological --propose` prints + the `words.yaml` patch for each mismatch and writes nothing. Read every row against the rules before accepting it. The + `batch` command of `src/SIL.Machine.Morphology.HermitCrab.Tool` records the same signatures for a word list. + +## Running another engine + +`--adapter ""` runs the same fixtures through an external engine instead of the C# engine. The +template placeholders are `{grammar}`, `{words}`, `{output}` and `{morpher}`; the engine writes one row per word to +`{output}` in the format of `batch` in `src/SIL.Machine.Morphology.HermitCrab.Tool`, as the suite's `PROTOCOL.md` +describes. Pass `--capabilities phonology` (the default in adapter mode is none) so fixtures the engine can run are not +skipped. For example, with a built PanGloss: + +``` + --fixtures conformance --capabilities phonology --adapter "/path/to/pangloss batch {grammar} {words} {output}" +``` + +The exit code is 0 when every run passes, 1 on a failure and 2 on a harness error; each failure is a `[FAIL] ` line. + +## Before you push + +Run each command from the repository root and read the output. Do not push on a partial run. + +``` +cat conformance.commit +git -C conformance rev-parse HEAD +pwsh ./scripts/comment-hygiene.ps1 +dotnet csharpier check . +dotnet build tests/SIL.Machine.Morphology.HermitCrab.Tests -c Release + --fixtures conformance --include-pathological + --fixtures conformance/stress --include-pathological + --fixtures conformance --check-manifest + --fixtures conformance --coverage-report +dotnet test tests/SIL.Machine.Morphology.HermitCrab.Tests -c Release +``` + +The two commits on the first line must match. `./local_check.sh --agent-strict` runs the repository-wide version of the +same checks, including that the pin is on the suite's `main`. + +In `conformance/`, `edge-cases/deep-optional-affix-nesting` carries a `budget_ms` of 15 s, and each stress fixture carries +60 to 120 s. Run Release builds; Debug is markedly slower. A loaded machine can still exceed one; rerun on a quiet +machine before concluding the engine regressed, and do not raise a budget here. Budgets are the suite's. From 7d18eabd6c49b30eb79be9bfa613578dbb92a8a6 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:10:48 -0500 Subject: [PATCH 07/15] Stop a rejected batch line leaking --start into the next The dispatcher counted a batch line's arguments after it had parsed --start, and the reset ran only when Run completed. A script line such as "batch --start=2 words.txt" was rejected for its missing output path and left the next batch line resuming at index 2. BatchCommand now counts its own arguments and clears the start index before it rejects a line. Co-Authored-By: Claude Sonnet 5.5 --- .../BatchCommand.cs | 17 +++++++++- .../BatchCommandTests.cs | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs index 2eaa313c7..2835c8713 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs @@ -14,6 +14,8 @@ namespace SIL.Machine.Morphology.HermitCrab; /// internal class BatchCommand : ConsoleCommand { + private const string UsageArguments = " "; + private readonly HCContext _context; private int _startIndex; @@ -23,7 +25,7 @@ public BatchCommand(HCContext context) IsCommand("batch", "Parses every word in a word list, recording a result signature and timing per word"); SkipsCommandSummaryBeforeRunning(); - HasAdditionalArguments(2, " "); + AllowsAnyAdditionalArguments(UsageArguments); HasOption("start=", "0-based line index to resume at (for crash recovery)", v => _startIndex = ParseStart(v)); } @@ -35,6 +37,19 @@ private static int ParseStart(string value) return start; } + /// + /// Counts the arguments here rather than with HasAdditionalArguments, which the dispatcher checks after + /// parsing --start. One instance serves every line of a script, so a line rejected there would leave + /// --start set for the next line. + /// + public override int? OverrideAfterHandlingArgumentsBeforeRun(string[] remainingArguments) + { + if (remainingArguments.Length == 2) + return null; + _startIndex = 0; + throw new ConsoleHelpAsException($"Invalid number of arguments - expected: {UsageArguments}"); + } + public override int Run(string[] remainingArguments) { string wordListPath = remainingArguments[0]; diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs index 510238842..dfcbdafb8 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs @@ -41,6 +41,40 @@ public void Run_AfterAResumedRun_StartsTheNextRunAtTheFirstWord() } } + [Test] + public void Run_AfterARejectedLineWithStart_StartsTheNextRunAtTheFirstWord() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a", "b", "c"]); + try + { + string freshPath = Path.Combine(directory, "fresh.tsv"); + // The output path is missing, so the dispatcher rejects the line after it has parsed --start. + int rejected = ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", "--start=2", wordsPath], + TextWriter.Null + ); + ConsoleCommandDispatcher.DispatchCommand(commands, ["batch", wordsPath, freshPath], TextWriter.Null); + + Assert.Multiple(() => + { + Assert.That(rejected, Is.Not.EqualTo(0)); + Assert.That(File.ReadAllLines(freshPath)[0], Does.StartWith("0\t")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + [TestCase("-1")] [TestCase("x")] [TestCase("+1")] From 61213b6b5403d5d9afb044116ac38bd99fc28f0e Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:12:41 -0500 Subject: [PATCH 08/15] Count only an engine crash as a match for expect_crash in self-check Self-check passed an expect_crash fixture on any exception except the watchdog timeout, so a harness failure such as the rule index rejecting a duplicate rule name read as the expected crash. Adapter mode already credited only EngineCrashException. The engine load and the parse now throw that exception, and the crash match requires it. Co-Authored-By: Claude Sonnet 5.5 --- .../Runner.cs | 23 ++++++++++-- .../ConformanceHarnessTests.cs | 37 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs index f3a283e82..09363ef34 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs @@ -151,8 +151,9 @@ TextWriter proposeOutput { sw.Stop(); result.ElapsedMs = sw.ElapsedMilliseconds; - // The watchdog's own timeout is never a crash match, as in adapter mode. - if (fixture.Words.ExpectCrash && ex is not TimeoutException) + // As in adapter mode, only the engine's own crash matches expect_crash; a timeout or a harness + // failure is a failure. + if (fixture.Words.ExpectCrash && ex is EngineCrashException) { result.Outcome = FixtureOutcome.Passed; result.Reason = $"engine crashed as expected (words.yaml expect_crash: true): {ex.Message}"; @@ -215,8 +216,18 @@ private static void RunAllWords( CancellationToken cancellationToken ) { - Language language = XmlLanguageLoader.Load(fixture.GrammarPath); - Morpher morpher = ConformanceMorpherFactory.CreateTracing(language, run.Morpher); + // Only a failure of the engine itself is an EngineCrashException, so expect_crash cannot excuse a + // harness failure such as the rule index rejecting a grammar. + Morpher morpher; + try + { + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + morpher = ConformanceMorpherFactory.CreateTracing(language, run.Morpher); + } + catch (Exception ex) + { + throw new EngineCrashException($"engine failed to load '{fixture.GrammarPath}': {ex.Message}", ex); + } GrammarRuleIndex ruleIndex = GrammarRuleIndex.Load(fixture.GrammarPath); foreach (WordEntry word in run.Words) @@ -235,6 +246,10 @@ CancellationToken cancellationToken wasSkipped = true; results = new List(); } + catch (Exception ex) + { + throw new EngineCrashException($"engine failed to parse '{word.Word}': {ex.Message}", ex); + } string actualSignature = SignatureFormat.BuildSignature(results); string expectedSignature = FixtureMaterializer.BuildExpectedSignature(word); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs index 697b33514..aa4624125 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs @@ -154,6 +154,43 @@ public void RunSelfCheck_TimedOutFixture_ReportsNoWordResults() }); } + [Test] + public void RunSelfCheck_ExpectCrashFixtureWhoseGrammarTheHarnessRejects_Fails() + { + string root = Path.Combine(Path.GetTempPath(), $"harness-failure-{Guid.NewGuid():N}"); + string source = Path.Combine(ConformanceSuite.Root(), "edge-cases", "epenthesis-fed-merge"); + string target = Path.Combine(root, "edge-cases", "epenthesis-fed-merge"); + Directory.CreateDirectory(target); + foreach (string file in Directory.GetFiles(source)) + File.Copy(file, Path.Combine(target, Path.GetFileName(file))); + string wordsPath = Path.Combine(target, "words.yaml"); + string words = File.ReadAllText(wordsPath); + Assert.That(words, Does.Contain("\nwords:\n")); + File.WriteAllText(wordsPath, words.Replace("\nwords:\n", "\nexpect_crash: true\nwords:\n")); + // The engine loads this grammar, but the harness's rule index rejects two rules with one name. + string grammarPath = Path.Combine(target, "grammar.xml"); + string grammar = File.ReadAllText(grammarPath); + Assert.That(grammar, Does.Contain("mergeTi")); + File.WriteAllText(grammarPath, grammar.Replace("mergeTi", "insertTiAfterBAtEnd")); + try + { + RunReport report = Runner.RunSelfCheck( + Fixture.DiscoverAll(root), + includePathological: true, + engineCapabilities: Runner.DefaultCapabilities, + propose: false, + proposeOutput: TextWriter.Null + ); + + FixtureResult result = report.Results.Single(); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + [Test] public void RunAdapter_ExpectCrashFixture_FailsWhenTheEngineAlsoCrashesOnAnOrdinaryFixture() { From 6451c04fe7378ee177d7a57fd329781ededd30ba Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:14:07 -0500 Subject: [PATCH 09/15] Name the fixture when --check-manifest cannot read its grammar A grammar that was not well-formed XML made GrammarValidation throw from the reader, and --check-manifest ended in an unhandled exception whose stack trace did not name the fixture. The loop now reports the fixture id and the reader's message and exits 2, as it does for a grammar that fails the DTD. Co-Authored-By: Claude Sonnet 5.5 --- .../Program.cs | 18 ++++++++++---- .../ConformanceCommandLineTests.cs | 24 +++++++++++++++++++ .../ConformanceManifestTests.cs | 2 +- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs index 98dc5054e..87630d9a4 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Xml; namespace SIL.Machine.Morphology.HermitCrab.Conformance; @@ -224,10 +225,19 @@ private static int RunCheckManifest(string suiteRoot) // Validate every fixture before comparing, so a rejected one is named rather than reported as stale. foreach (ManifestFixture fixture in corpus.Fixtures) { - IReadOnlyList messages = GrammarValidation.Validate( - Path.Combine(suiteRoot, fixture.GrammarPath.Replace('/', Path.DirectorySeparatorChar)), - dtd - ); + IReadOnlyList messages; + try + { + messages = GrammarValidation.Validate( + Path.Combine(suiteRoot, fixture.GrammarPath.Replace('/', Path.DirectorySeparatorChar)), + dtd + ); + } + catch (Exception ex) when (ex is XmlException or IOException or UnauthorizedAccessException) + { + Console.Error.WriteLine($"{fixture.FixtureId}: grammar cannot be read: {ex.Message}"); + return 2; + } if (messages.Count != 0) { Console.Error.WriteLine($"{fixture.FixtureId}: grammar does not validate against {corpus.DtdPath}"); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs index 74ee6c2b1..3148f00e7 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs @@ -114,4 +114,28 @@ public void CoverageReport_RefusesAnOutputInsideTheFixturesRoot() Assert.That(Directory.Exists(inside), Is.False); }); } + + [Test] + [NonParallelizable] + public void CheckManifest_MalformedGrammar_NamesTheFixtureAndFails() + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = ConformanceManifestTests.CreateMiniSuite(FixtureId); + try + { + File.WriteAllText(Path.Combine(root, "edge-cases", "strrep-identity", "grammar.xml"), " + { + Assert.That(exitCode, Is.EqualTo(2), error); + Assert.That(error, Does.Contain(FixtureId)); + }); + } + finally + { + Directory.Delete(root, recursive: true); + } + } } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs index db3c8f2f2..5347421e6 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs @@ -176,7 +176,7 @@ public void PlainYamlOnlyIsEnforced(string _, string yaml) } /// A minimal suite carrying one real fixture, for mutation tests. - private static string CreateMiniSuite(string fixtureId) + internal static string CreateMiniSuite(string fixtureId) { string source = SuiteRoot(); string root = Path.Combine(Path.GetTempPath(), $"hc-manifest-suite-{Guid.NewGuid():N}"); From caff4ae16feab374658a76353fb09312dc9a7c7f Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:15:13 -0500 Subject: [PATCH 10/15] Let the stress parity test pass for a fixture with variants RunSelfCheck adds one result per word run, so a stress fixture with a variants list reported two passes and failed the test's expectation of exactly one. The test now requires at least one result and every result to have passed. No current stress fixture has variants. Co-Authored-By: Claude Sonnet 5.5 --- .../StressFixtureParityTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs index 9e420e8e3..3d85747a6 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs @@ -25,6 +25,8 @@ public void ParsesMatchTheMasterEngine(string fixtureId) ); string reasons = string.Join("; ", report.Results.Select(r => r.Reason)); Assert.That(report.Failed, Is.EqualTo(0), reasons); - Assert.That(report.Passed, Is.EqualTo(1), reasons); + // A fixture with variants reports one result per variant, so pass means every run passed. + Assert.That(report.Results, Is.Not.Empty, reasons); + Assert.That(report.Passed, Is.EqualTo(report.Results.Count), reasons); } } From c72e61fef5c5b0a4385ebf4d24cb3e7f14d5753d Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:15:42 -0500 Subject: [PATCH 11/15] Compare the suite before and after the runner steps in local_check local_check reported suite-modified whenever the suite had uncommitted edits, which the targets allow and HC_CONFORMANCE_SUITE is meant for, and it missed files the runner left untracked. It now records the suite's status and diff before the runner steps and fails only if they differ afterwards. Co-Authored-By: Claude Sonnet 5.5 --- local_check.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/local_check.sh b/local_check.sh index b46c37342..671e041d4 100755 --- a/local_check.sh +++ b/local_check.sh @@ -40,14 +40,21 @@ dotnet csharpier check . || failed+=(csharpier) dotnet build --no-restore -c Release || failed+=(build) dotnet test --verbosity normal || failed+=(test) +# What the suite looks like before the runner steps, so edits you made to it do not read as the runner's. +suite_state() { + git -C "$suite" status --porcelain --untracked-files=all 2>/dev/null + git -C "$suite" diff 2>/dev/null +} +suite_before=$(suite_state | sha1sum) + # The same checks as .github/workflows/conformance.yml. hc_conformance=(dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --) "${hc_conformance[@]}" --fixtures "$suite" --include-pathological || failed+=(conformance) "${hc_conformance[@]}" --fixtures "$suite/stress" --include-pathological || failed+=(conformance-stress) "${hc_conformance[@]}" --fixtures "$suite" --check-manifest || failed+=(manifest) "${hc_conformance[@]}" --fixtures "$suite" --coverage-report || failed+=(coverage-report) -# Nothing above may have changed the suite clone. -git -C "$suite" diff --exit-code || failed+=(suite-modified) +# The runner steps must not have changed the suite, tracked or untracked. +[ "$(suite_state | sha1sum)" = "$suite_before" ] || failed+=(suite-modified) # The pin must be on the suite's main (CI checks the same). Not checked without a network or with another suite. if [ -z "${HC_CONFORMANCE_SUITE:-}" ]; then if git -C conformance fetch --quiet origin main; then From 4d41c1d1e090b689f0cabb4e1ce61b47c4ad9fce Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:16:24 -0500 Subject: [PATCH 12/15] Sort capability and key names with the ordinal comparer Three diagnostic messages sorted names with the current culture, so the order could differ between machines. They now use StringComparer.Ordinal as the rest of the runner does. Only the message text is affected. Co-Authored-By: Claude Sonnet 5.5 --- .../MaterializedRunner.cs | 2 +- src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs | 2 +- .../WordsYamlLoader.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs index 98a9549ab..dc1829c6e 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs @@ -74,7 +74,7 @@ bool includePathological result.Outcome = FixtureOutcome.Skipped; result.Reason = $"requires [{string.Join(",", fixture.Manifest.Requires)}], " - + $"engine declares [{string.Join(",", engine.Capabilities.OrderBy(c => c))}]; " + + $"engine declares [{string.Join(",", engine.Capabilities.OrderBy(c => c, StringComparer.Ordinal))}]; " + $"missing [{string.Join(",", missingCapabilities)}]"; report.Results.Add(result); continue; diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs index 87630d9a4..3d8a9a1a8 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs @@ -163,7 +163,7 @@ private static void PrintRunReport(RunReport report, IEngine engine) { Console.WriteLine(); Console.WriteLine( - $"engine: {engine.Name}, capabilities: [{string.Join(",", engine.Capabilities.OrderBy(c => c))}]" + $"engine: {engine.Name}, capabilities: [{string.Join(",", engine.Capabilities.OrderBy(c => c, StringComparer.Ordinal))}]" ); if (report.ExcludedPathologicalCount > 0) { diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs index ae3aea6bb..78a182ea5 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs @@ -457,7 +457,7 @@ string context { throw new WordsYamlException( path, - $"{context}: unknown key '{key}' (allowed: {string.Join(", ", allowedKeys.OrderBy(k => k))})" + $"{context}: unknown key '{key}' (allowed: {string.Join(", ", allowedKeys.OrderBy(k => k, StringComparer.Ordinal))})" ); } if (map.ContainsKey(key)) From ad9765b3b409ea04a31fc2e05e6b203e13aa1e3b Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:16:25 -0500 Subject: [PATCH 13/15] Say how the adapter command template treats quotes The template is split by the same routine as an hc script line, which drops quote characters and keeps no empty token, so --grammar="{grammar}" reached the engine as two arguments. The README now says to write --grammar {grammar}, and that placeholders are filled after the split so a path with spaces stays one argument. Co-Authored-By: Claude Sonnet 5.5 --- src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md index 62f309a00..1f0e6e3ca 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md @@ -73,7 +73,10 @@ The suite checks in files that are computed from its fixtures. The runner only c template placeholders are `{grammar}`, `{words}`, `{output}` and `{morpher}`; the engine writes one row per word to `{output}` in the format of `batch` in `src/SIL.Machine.Morphology.HermitCrab.Tool`, as the suite's `PROTOCOL.md` describes. Pass `--capabilities phonology` (the default in adapter mode is none) so fixtures the engine can run are not -skipped. For example, with a built PanGloss: +skipped. The template is split at spaces into the program and its arguments, the same way as a line of an +`hc --script-file`; a quote only groups words and is not passed on, so write `--grammar {grammar}`, not +`--grammar="{grammar}"`. Each placeholder is filled in after the split, so a path with spaces still reaches the engine +as one argument. For example, with a built PanGloss: ``` --fixtures conformance --capabilities phonology --adapter "/path/to/pangloss batch {grammar} {words} {output}" From 1bd1dc9d63bd2f74a6ca31d263a274e046ca3dd5 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 15:21:22 -0500 Subject: [PATCH 14/15] Remove SelfCheckEngine, whose Run had no caller Self-check mode runs through Runner.RunSelfCheck, and RunAdapter is reached only with an AdapterEngine, so SelfCheckEngine.Run was never called. Without it the class only carried a name and a capability set into the report header, so Program now passes those to PrintRunReport directly. Comments that named it as an IEngine are corrected. Co-Authored-By: Claude Sonnet 5.5 --- .../IEngine.cs | 7 +- .../MaterializedFixture.cs | 2 +- .../MaterializedRunner.cs | 4 +- .../Program.cs | 26 +++---- .../Runner.cs | 4 +- .../SelfCheckEngine.cs | 69 ------------------- 6 files changed, 21 insertions(+), 91 deletions(-) delete mode 100644 src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs index 7fbeed32a..5a603ae94 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs @@ -4,13 +4,12 @@ namespace SIL.Machine.Morphology.HermitCrab.Conformance; public interface IEngine { - /// A short, human-readable name for reporting (e.g. "self-check", "adapter"). + /// A short, human-readable name for reporting (e.g. "adapter"). string Name { get; } /// - /// The capability set this engine declares support for. Self-check mode always reports the - /// full set (it IS the reference C# oracle); an adapter's set comes from the --capabilities - /// flag. + /// The capability set this engine declares support for; an adapter's set comes from the + /// --capabilities flag. /// IReadOnlySet Capabilities { get; } diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs index b22bb02c2..4b637fc10 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs @@ -5,7 +5,7 @@ namespace SIL.Machine.Morphology.HermitCrab.Conformance; /// /// A fixture materialized onto disk in the on-disk adapter shape (grammar.xml + words.txt + /// expected.tsv + an in-memory ), so it can be handed to -/// and (self-check or an external adapter +/// and (an external adapter /// process). Built by from a . /// public class MaterializedFixture( diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs index dc1829c6e..a76d7d706 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs @@ -200,8 +200,8 @@ bool includePathological /// Runs the engine for one fixture. If the manifest carries a budget, engine.Run happens on a /// background thread and this waits up to budget + a small margin; AdapterEngine independently /// plumbs the same budget into its own process timeout (so a subprocess adapter gets killed), - /// but self-check's in-process call has no external process to kill, so this watchdog is what - /// keeps a runaway self-check parse from hanging the whole harness run. If the watchdog expires, + /// but an in-process engine has no external process to kill, so this watchdog is what + /// keeps a runaway engine from hanging the whole harness run. If the watchdog expires, /// a TimeoutException is thrown and the background thread is simply abandoned -- this is a /// short-lived CLI, so a leaked runaway thread on the rare budget-blown fixture is an acceptable /// trade for keeping this shape simple. diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs index 3d8a9a1a8..921232739 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs @@ -109,23 +109,23 @@ private static int Main(string[] args) return RunCoverageReport(fixtures, fixturesRoot, constructsPath, coverageOutput); } - IEngine engine; + RunReport report; + string engineName; + IReadOnlySet engineCapabilities; if (adapterTemplate != null) { - HashSet capabilities = ParseCapabilities(capabilitiesArg ?? ""); - engine = new AdapterEngine(adapterTemplate, capabilities); + var engine = new AdapterEngine(adapterTemplate, ParseCapabilities(capabilitiesArg ?? "")); + report = Runner.RunAdapter(fixtures, engine, includePathological); + engineName = engine.Name; + engineCapabilities = engine.Capabilities; } else { - IReadOnlySet capabilities = capabilitiesProvided ? ParseCapabilities(capabilitiesArg) : null; - engine = new SelfCheckEngine(capabilities); + engineName = "self-check"; + engineCapabilities = capabilitiesProvided ? ParseCapabilities(capabilitiesArg) : Runner.DefaultCapabilities; + report = Runner.RunSelfCheck(fixtures, includePathological, engineCapabilities, propose, Console.Out); } - - RunReport report = - adapterTemplate != null - ? Runner.RunAdapter(fixtures, engine, includePathological) - : Runner.RunSelfCheck(fixtures, includePathological, engine.Capabilities, propose, Console.Out); - PrintRunReport(report, engine); + PrintRunReport(report, engineName, engineCapabilities); bool anyRan = report.Passed > 0 || report.Failed > 0; if (!anyRan) @@ -159,11 +159,11 @@ private static HashSet ParseCapabilities(string arg) .ToHashSet(StringComparer.Ordinal); } - private static void PrintRunReport(RunReport report, IEngine engine) + private static void PrintRunReport(RunReport report, string engineName, IReadOnlySet capabilities) { Console.WriteLine(); Console.WriteLine( - $"engine: {engine.Name}, capabilities: [{string.Join(",", engine.Capabilities.OrderBy(c => c, StringComparer.Ordinal))}]" + $"engine: {engineName}, capabilities: [{string.Join(",", capabilities.OrderBy(c => c, StringComparer.Ordinal))}]" ); if (report.ExcludedPathologicalCount > 0) { diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs index 09363ef34..47a131740 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs @@ -22,8 +22,8 @@ public class RunReport /// /// Runs fixtures under conformance/languages and conformance/edge-cases. Self-check mode is a /// custom, in-process loop (it needs per-word signature-SET comparison against words.yaml and -/// traced-rules verification, neither of which / -/// does). Adapter mode instead materializes each fixture onto disk and hands it to +/// traced-rules verification, neither of which does). Adapter mode instead +/// materializes each fixture onto disk and hands it to /// + -- see . /// public static class Runner diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs deleted file mode 100644 index d900006c0..000000000 --- a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SelfCheckEngine.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace SIL.Machine.Morphology.HermitCrab.Conformance; - -/// -/// Runs a fixture against the live C# HermitCrab engine, in-process: loads grammar.xml via -/// , builds a , parses every word in -/// words.txt, and computes the exact same signature that -/// BatchCommand's oracle uses -- byte-identical, since it's the same shared code, not a -/// reimplementation. This is what catches a fixture going stale when the oracle itself changes. -/// -/// -/// Lets a caller override the reference engine's normal full-capability declaration -- e.g. -/// --capabilities "", which is how the harness's own capability-filtering skip path gets -/// exercised against the live oracle without needing a second, capability-limited engine on hand. -/// Null (the default) means the full set. -/// -public class SelfCheckEngine(IReadOnlySet capabilities = null) : IEngine -{ - public string Name => "self-check"; - - // The reference C# engine implements every capability the suite currently knows about, unless - // overridden above. - public IReadOnlySet Capabilities { get; } = capabilities ?? Runner.DefaultCapabilities; - - public List Run(MaterializedFixture fixture) - { - Language language; - Morpher morpher; - try - { - language = XmlLanguageLoader.Load(fixture.GrammarPath); - morpher = ConformanceMorpherFactory.Create(language, fixture.Manifest.Morpher); - } - catch (Exception ex) - { - throw new EngineCrashException( - $"self-check engine crashed loading grammar for fixture '{fixture.Id}': {ex.Message}", - ex - ); - } - - string[] words = SignatureFormat.LoadWords(fixture.WordsPath); - - var rows = new List(words.Length); - for (int i = 0; i < words.Length; i++) - { - string word = words[i]; - try - { - (string status, _, string signature) = SignatureFormat.ParseOneWord( - morpher, - word, - fixture.Manifest.Morpher.GuessRoots ?? false - ); - rows.Add(new TsvRow(word, status, signature)); - } - catch (Exception ex) - { - throw new EngineCrashException( - $"self-check engine crashed parsing word '{word}' in fixture '{fixture.Id}': {ex.Message}", - ex - ); - } - } - return rows; - } -} From 4d9b454d42a0da35e058641a65596339a79d115b Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 2 Nov 2026 16:24:58 -0500 Subject: [PATCH 15/15] Format ConformanceCommandLineTests with csharpier Co-Authored-By: Claude Haiku 5.5 --- .../ConformanceCommandLineTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs index 3148f00e7..1c5a163e5 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs @@ -123,7 +123,10 @@ public void CheckManifest_MalformedGrammar_NamesTheFixtureAndFails() string root = ConformanceManifestTests.CreateMiniSuite(FixtureId); try { - File.WriteAllText(Path.Combine(root, "edge-cases", "strrep-identity", "grammar.xml"), "