diff --git a/.csharpierignore b/.csharpierignore index b5df929f0..e8b74c9a6 100644 --- a/.csharpierignore +++ b/.csharpierignore @@ -2,4 +2,6 @@ *.csproj *.props *.targets -*.xml \ No newline at end of file +*.xml +/conformance/ +/conformance.tmp-*/ diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..aec98da25 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,84 @@ +name: Conformance + +on: + push: + branches: + - "**" + paths: + - "conformance.commit" + - "src/SIL.Machine/**" + - "src/SIL.Machine.Morphology.HermitCrab*/**" + - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**" + - ".github/workflows/conformance.yml" + pull_request: + paths: + - "conformance.commit" + - "src/SIL.Machine/**" + - "src/SIL.Machine.Morphology.HermitCrab*/**" + - "tests/SIL.Machine.Morphology.HermitCrab.Tests/**" + - ".github/workflows/conformance.yml" + +jobs: + conformance: + name: Conformance suite + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 10.0.x + + # Building the runner clones sillsdev/fieldworks-parser-conformance into conformance/ at the commit in + # conformance.commit; see EnsureConformanceSuite.targets. + # + # Release, because edge-cases/deep-optional-affix-nesting has a 15 s wall-clock budget that Debug uses + # most of on a small runner. Built once; every step below runs the built runner. + - name: Build the runner + run: dotnet build -c Release src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj + + # Every fixture's words.yaml must match what the C# engine produces, in process, including the fixtures + # with a budget_ms. + - name: Run conformance self-check + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --include-pathological + + # The suite's stress fixtures (conformance/stress) must agree with the engine too. + - name: Run stress self-check + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance/stress --include-pathological + + # The suite's generated manifest must match a fresh computation, and every grammar and words.yaml must + # validate against the suite's DTD and schema. + - name: Check the suite manifest + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --check-manifest + + # Exits 1 on an uncovered construct, on a dead rule (exercised by zero words) missing from + # conformance/dead-rules-allowed.tsv, on an allowed rule some word now exercises, and when the suite's + # coverage.csv or rules.csv is stale. It writes nothing; the suite owns both files. + - name: Run coverage report + if: always() + run: >- + dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance -- + --fixtures conformance --coverage-report + + # A pin that is not on the suite's main could be lost when that branch is deleted. + - name: Check the pinned commit is on the suite's main + if: always() + run: | + pin="$(tr -d '[:space:]' < conformance.commit)" + git -C conformance fetch --quiet origin main + if ! git -C conformance merge-base --is-ancestor "$pin" origin/main; then + echo "::error file=conformance.commit::$pin is not an ancestor of origin/main in the conformance suite" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 0437b6304..b7a33c859 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ tests/SIL.Machine.Tests/Corpora/TestData/pretranslations.json # Local, generated review summaries .review/ + +# The conformance suite, cloned here at the commit in conformance.commit by the build +/conformance/ +/conformance.tmp-*/ diff --git a/AGENTS.md b/AGENTS.md index 6e2e7194d..1127b8480 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,3 +78,11 @@ first row wins: `docs/review/devils-advocate.md` is an optional adversarial pass for a high-risk change. Add a nested `AGENTS.md` only when a subtree needs different rules. + +`conformance/` is an ignored clone of sillsdev/fieldworks-parser-conformance at +the commit in `conformance.commit`; do not edit it from machine. Building the +HermitCrab tests fetches it and moves it to the pinned commit after a pull or +branch switch. Changes to the grammars land in that repository, following its +`AGENTS.md`. To try an unmerged suite change, set `HC_CONFORMANCE_SUITE=`. +How to get, bump and check the suite from machine is in +`src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md`. diff --git a/Machine.sln b/Machine.sln index 0d91d2dac..eb4ead14e 100644 --- a/Machine.sln +++ b/Machine.sln @@ -45,6 +45,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.Se EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.SentencePiece.Tests", "tests\SIL.Machine.Tokenization.SentencePiece.Tests\SIL.Machine.Tokenization.SentencePiece.Tests.csproj", "{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Morphology.HermitCrab.Conformance", "src\SIL.Machine.Morphology.HermitCrab.Conformance\SIL.Machine.Morphology.HermitCrab.Conformance.csproj", "{8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -99,6 +101,10 @@ Global {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Debug|Any CPU.Build.0 = Debug|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.Build.0 = Release|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -116,6 +122,7 @@ Global {0BBEED18-3CC1-4BA6-8E11-603A5D2DEDC9} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {09135223-6285-4268-BFA3-147BDB5B663B} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC} = {66246A1C-8D45-40FB-A660-C58577122CA7} + {8E40C2C5-F4C5-4A41-B78A-10A43FCF0CE3} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/README.md b/README.md index 2ad4023a9..1113d822c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Machine provides implementations of many common statistical word alignment model Machine contains a rule-based morphological/phonological parser called HermitCrab. +The HermitCrab conformance grammars are in [sillsdev/fieldworks-parser-conformance](https://github.com/sillsdev/fieldworks-parser-conformance), an engine-neutral suite of grammars a FieldWorks project can produce, each with the parses an engine must return. Machine runs them from a clone in `conformance/`, at the commit named in `conformance.commit`, which the build fetches (`dotnet build` or `dotnet test` of the HermitCrab tests), with the runner in [`src/SIL.Machine.Morphology.HermitCrab.Conformance`](src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md). + ### Feature Structures Machine provides a flexible implementation of feature structures with efficient unification, subsumption, and priority union operations. Feature values can be atomic symbols, strings, or variables. diff --git a/conformance.commit b/conformance.commit new file mode 100644 index 000000000..f4acce70d --- /dev/null +++ b/conformance.commit @@ -0,0 +1 @@ +7a0f7419fc706425171080ddbe3db263197b2d0d diff --git a/local_check.sh b/local_check.sh index e80ed7ff3..671e041d4 100755 --- a/local_check.sh +++ b/local_check.sh @@ -30,12 +30,40 @@ fi # output read through tail cannot pass off a build failure as a test pass. failed=() +# conformance/ is the suite clone at the commit in conformance.commit; the build below clones it or moves it to the +# pin. HC_CONFORMANCE_SUITE= uses another copy. +suite="${HC_CONFORMANCE_SUITE:-conformance}" + dotnet tool restore || failed+=(tool-restore) dotnet restore || failed+=(restore) dotnet csharpier check . || failed+=(csharpier) dotnet build --no-restore -c Release || failed+=(build) dotnet test --verbosity normal || failed+=(test) +# What the suite looks like before the runner steps, so edits you made to it do not read as the runner's. +suite_state() { + git -C "$suite" status --porcelain --untracked-files=all 2>/dev/null + git -C "$suite" diff 2>/dev/null +} +suite_before=$(suite_state | sha1sum) + +# The same checks as .github/workflows/conformance.yml. +hc_conformance=(dotnet run --no-build -c Release --project src/SIL.Machine.Morphology.HermitCrab.Conformance --) +"${hc_conformance[@]}" --fixtures "$suite" --include-pathological || failed+=(conformance) +"${hc_conformance[@]}" --fixtures "$suite/stress" --include-pathological || failed+=(conformance-stress) +"${hc_conformance[@]}" --fixtures "$suite" --check-manifest || failed+=(manifest) +"${hc_conformance[@]}" --fixtures "$suite" --coverage-report || failed+=(coverage-report) +# The runner steps must not have changed the suite, tracked or untracked. +[ "$(suite_state | sha1sum)" = "$suite_before" ] || failed+=(suite-modified) +# The pin must be on the suite's main (CI checks the same). Not checked without a network or with another suite. +if [ -z "${HC_CONFORMANCE_SUITE:-}" ]; then + if git -C conformance fetch --quiet origin main; then + git -C conformance merge-base --is-ancestor "$(tr -d '[:space:]' < conformance.commit)" origin/main || failed+=(pin-not-on-main) + else + echo "local_check: could not fetch the suite's main, so the pin was not checked against it" + fi +fi + if [ "${#failed[@]}" -ne 0 ]; then echo "local_check: FAILED (${failed[*]})" exit 1 diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs new file mode 100644 index 000000000..c6c1ebfba --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/AdapterEngine.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public class AdapterEngine : IEngine +{ + private readonly List _templateTokens; + private readonly int _timeoutMs; + + public AdapterEngine(string commandTemplate, IReadOnlySet capabilities, int timeoutMs = 5 * 60 * 1000) + { + _templateTokens = new List(SIL.Machine.Morphology.HermitCrab.Program.SplitCommandLine(commandTemplate)); + if (_templateTokens.Count == 0) + throw new InvalidOperationException("empty --adapter command template"); + _timeoutMs = timeoutMs; + Capabilities = capabilities; + } + + public string Name => "adapter"; + + public IReadOnlySet Capabilities { get; } + + public List Run(MaterializedFixture fixture) + { + string outputPath = Path.Combine(Path.GetTempPath(), "hc-conformance-" + Guid.NewGuid().ToString("N") + ".tsv"); + try + { + // A fixture with a morpher: block reaches an adapter only through {morpher}; without it the + // adapter would run at the defaults the fixture overrides. + if (!fixture.Manifest.Morpher.IsEmpty && !_templateTokens.Exists(t => t.Contains("{morpher}"))) + { + throw new InvalidOperationException( + $"fixture '{fixture.Id}' sets morpher: settings but the --adapter command template has no " + + "{morpher} placeholder to pass them through; add it or drop the morpher-settings capability" + ); + } + var tokens = new List(_templateTokens.Count); + foreach (string token in _templateTokens) + { + tokens.Add( + token + .Replace("{grammar}", Path.GetFullPath(fixture.GrammarPath)) + .Replace("{words}", Path.GetFullPath(fixture.WordsPath)) + .Replace("{output}", outputPath) + .Replace("{morpher}", Path.GetFullPath(fixture.MorpherPath)) + ); + } + + var psi = ChildProcessEnvironment.CreateStartInfo(tokens[0]); + psi.RedirectStandardOutput = true; + psi.RedirectStandardError = true; + for (int i = 1; i < tokens.Count; i++) + psi.ArgumentList.Add(tokens[i]); + + using Process process = + Process.Start(psi) + ?? throw new InvalidOperationException($"failed to start adapter process '{tokens[0]}'"); + + Task stdoutTask = process.StandardOutput.ReadToEndAsync(); + Task stderrTask = process.StandardError.ReadToEndAsync(); + + int effectiveTimeoutMs = + fixture.Manifest.Budget != null ? (int)fixture.Manifest.Budget.WallClockMs : _timeoutMs; + + bool exited = process.WaitForExit(effectiveTimeoutMs); + if (!exited) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) { } + throw new TimeoutException( + $"adapter process for fixture '{fixture.Id}' did not exit within {effectiveTimeoutMs}ms" + ); + } + + string stdout = stdoutTask.GetAwaiter().GetResult(); + string stderr = stderrTask.GetAwaiter().GetResult(); + + if (process.ExitCode != 0) + { + throw new EngineCrashException( + $"adapter process exited {process.ExitCode} for fixture '{fixture.Id}'.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + } + if (!File.Exists(outputPath)) + { + throw new InvalidOperationException( + $"adapter process for fixture '{fixture.Id}' did not produce its output file.\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + } + return SignatureTsv.ReadFile(outputPath); + } + finally + { + try + { + if (File.Exists(outputPath)) + File.Delete(outputPath); + } + catch (IOException) { } + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs new file mode 100644 index 000000000..6cc548b26 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CanonicalJson.cs @@ -0,0 +1,100 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Produces compact UTF-8 JSON with recursively ordinal-sorted object keys. +internal static class CanonicalJson +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = null, + }; + + internal static byte[] SerializeUtf8(T value) + { + ArgumentNullException.ThrowIfNull(value); + using JsonDocument document = JsonDocument.Parse(JsonSerializer.SerializeToUtf8Bytes(value, SerializerOptions)); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false })) + WriteElement(document.RootElement, writer); + return stream.ToArray(); + } + + internal static string Serialize(T value) => Encoding.UTF8.GetString(SerializeUtf8(value)); + + private static void WriteElement(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + var properties = new List(); + var propertyNames = new HashSet(StringComparer.Ordinal); + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!propertyNames.Add(property.Name)) + throw new InvalidDataException($"Duplicate JSON property '{property.Name}'."); + properties.Add(property); + } + properties.Sort((left, right) => StringComparer.Ordinal.Compare(left.Name, right.Name)); + foreach (JsonProperty property in properties) + { + writer.WritePropertyName(property.Name); + WriteElement(property.Value, writer); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in element.EnumerateArray()) + WriteElement(item, writer); + writer.WriteEndArray(); + break; + case JsonValueKind.String: + writer.WriteStringValue(NormalizeString(element.GetString()!)); + break; + case JsonValueKind.Number: + if ( + !decimal.TryParse( + element.GetRawText(), + NumberStyles.Float, + CultureInfo.InvariantCulture, + out decimal decimalValue + ) + ) + { + throw new InvalidDataException( + $"JSON number '{element.GetRawText()}' is outside the supported canonical range." + ); + } + writer.WriteRawValue( + decimalValue.ToString("G29", CultureInfo.InvariantCulture), + skipInputValidation: true + ); + break; + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + default: + throw new InvalidDataException($"Unsupported JSON value kind '{element.ValueKind}'."); + } + } + + private static string NormalizeString(string value) => + value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs new file mode 100644 index 000000000..b7ff31da2 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ChildProcessEnvironment.cs @@ -0,0 +1,36 @@ +#nullable enable +using System.Diagnostics; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// The only place this assembly (and the test assembly it grants internals to) may construct a child +/// ; ProcessStartInfoConstructionGateTests fails naming any other +/// file that does. Xplat Code Coverage attaches a CLR profiler to the current process via +/// CORECLR_*/COR_* environment variables, which ProcessStartInfo inherits into a child by default, and +/// a profiled child that then crashes or is killed has been observed to destabilize the coverage +/// channel the parent test host shares -- reproduced twice as a trace-less test host crash under +/// `--collect:"Xplat Code Coverage"` before this factory existed. +/// +internal static class ChildProcessEnvironment +{ + private static readonly string[] ProfilerVariableNames = + { + "CORECLR_ENABLE_PROFILING", + "CORECLR_PROFILER", + "CORECLR_PROFILER_PATH", + "CORECLR_PROFILER_PATH_32", + "CORECLR_PROFILER_PATH_64", + "COR_ENABLE_PROFILING", + "COR_PROFILER", + "COR_PROFILER_PATH", + }; + + internal static ProcessStartInfo CreateStartInfo(string fileName) + { + var startInfo = new ProcessStartInfo { FileName = fileName, UseShellExecute = false }; + foreach (string name in ProfilerVariableNames) + startInfo.EnvironmentVariables.Remove(name); + return startInfo; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs new file mode 100644 index 000000000..39899895b --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestArtifact.cs @@ -0,0 +1,45 @@ +#nullable enable +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Provenance and integrity for the suite's conformance fixtures. +/// Deliberately not a second copy of the ground truth. words.yaml is the single +/// canonical authored format -- it carries the reasoning for each fixture in comments, which no +/// generated JSON can hold -- and grammar.xml is the authoritative grammar. This manifest +/// records what those files are, which format each conforms to, and the hash a consumer verifies +/// them against. +public sealed record ConformanceManifest( + string FormatVersion, + string WordsAuthoringFormatVersion, + string GrammarFormatVersion, + string DtdPath, + string DtdSha256, + string SourceHash, + IReadOnlyList Fixtures +) +{ + public const string ManifestFormat = "hc-conformance-manifest/v2"; + public const string WordsFormat = "hc-conformance-words/v1"; + public const string GrammarFormat = "sil-machine-hermit-crab-input-xml/v1"; +} + +/// Cases authored in words.yaml, so a consumer can detect a partial +/// read without parsing the file. +public sealed record ManifestFixture( + string FixtureId, + string Category, + string DisplayLanguage, + string GrammarPath, + string GrammarSha256, + string WordsPath, + string WordsSha256, + int CaseCount, + bool ExpectedCrash +); + +/// Canonical serialization of the manifest. +public static class ManifestJson +{ + public static string Serialize(ConformanceManifest manifest) => CanonicalJson.Serialize(manifest); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs new file mode 100644 index 000000000..b7d2627b0 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceManifestGenerator.cs @@ -0,0 +1,141 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Builds the from the fixtures under a suite root. +/// Nothing here interprets a grammar or restates a words file; each contributes a +/// suite-relative path and a hash. The authored files stay the only ground truth. +public static class ConformanceManifestGenerator +{ + /// The suite's manifest, relative to the suite root. The suite's generator writes it. + public const string ManifestRelativePath = "generated/hc-conformance-manifest.v2.json"; + + /// The DTD as the manifest names it, relative to the suite root. + public const string DtdRelativePath = "HermitCrabInput.dtd"; + + /// Files the manifest's own validity depends on, beyond the fixtures themselves, relative to + /// the suite root. + private static readonly string[] AdditionalSourceFiles = + { + "constructs.txt", + "parity-check.py", + "schema/words.schema.json", + "schema/conformance-manifest.schema.json", + }; + + public static ConformanceManifest Generate(string suiteRoot) + { + ArgumentException.ThrowIfNullOrWhiteSpace(suiteRoot); + string root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(suiteRoot)); + string dtdRelative = DtdRelativePath; + string dtdHash = HashFile(Path.Combine(root, dtdRelative.Replace('/', Path.DirectorySeparatorChar))); + + List fixtures = Fixture.DiscoverAll(root); + var mapped = fixtures + .Select(fixture => MapFixture(fixture, root)) + .OrderBy(fixture => fixture.FixtureId, StringComparer.Ordinal) + .ToArray(); + + return new ConformanceManifest( + ConformanceManifest.ManifestFormat, + ConformanceManifest.WordsFormat, + ConformanceManifest.GrammarFormat, + dtdRelative, + dtdHash, + SourceHash(root, mapped, dtdRelative, dtdHash), + mapped + ); + } + + private static ManifestFixture MapFixture(Fixture fixture, string root) + { + string fixtureId = RelativePath(fixture.Directory, root); + string grammarPath = fixtureId + "/grammar.xml"; + string wordsPath = fixtureId + "/words.yaml"; + WordsYaml words = fixture.Words; + + string[] duplicates = words + .Words.Select(word => word.Word) + .GroupBy(input => input, StringComparer.Ordinal) + .Where(group => group.Count() > 1) + .Select(group => group.Key) + .OrderBy(input => input, StringComparer.Ordinal) + .ToArray(); + if (duplicates.Length != 0) + { + throw new InvalidDataException( + $"Fixture '{fixtureId}' repeats input(s): {string.Join(", ", duplicates)}. Inputs must be unique within a fixture." + ); + } + + if (words.ExpectCrash && words.Words.Count != 1) + { + throw new InvalidDataException( + $"Fixture '{fixtureId}' declares expect_crash and must contain exactly one case, but has {words.Words.Count}." + ); + } + + return new ManifestFixture( + fixtureId, + fixtureId.StartsWith("languages/", StringComparison.Ordinal) ? "language" : "edge-case", + words.Language, + grammarPath, + HashFile(Path.Combine(root, grammarPath.Replace('/', Path.DirectorySeparatorChar))), + wordsPath, + HashFile(Path.Combine(root, wordsPath.Replace('/', Path.DirectorySeparatorChar))), + words.Words.Count, + words.ExpectCrash + ); + } + + private static string RelativePath(string path, string basePath) => + Path.GetRelativePath(basePath, path).Replace('\\', '/'); + + /// Hash of every file the manifest's validity rests on. + /// Path- and EOL-invariant: only suite-relative paths and LF-normalized text + /// participate, so relocating the checkout or changing line endings cannot move it. + private static string SourceHash( + string root, + IReadOnlyList fixtures, + string dtdRelative, + string dtdHash + ) + { + var text = new StringBuilder(); + text.Append(ConformanceManifest.ManifestFormat).Append('\n'); + text.Append(dtdRelative).Append('\0').Append(dtdHash).Append('\n'); + foreach (ManifestFixture fixture in fixtures) + { + text.Append(fixture.FixtureId) + .Append('\0') + .Append(fixture.GrammarSha256) + .Append('\0') + .Append(fixture.WordsSha256) + .Append('\n'); + } + + foreach (string relative in AdditionalSourceFiles.OrderBy(path => path, StringComparer.Ordinal)) + { + string absolute = Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)); + text.Append(relative) + .Append('\0') + .Append(File.Exists(absolute) ? HashFile(absolute) : "") + .Append('\n'); + } + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))).ToLowerInvariant(); + } + + /// SHA-256 of a file's text with line endings normalized to LF. + internal static string HashFile(string path) + { + string content = File.ReadAllText(path).Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(content))).ToLowerInvariant(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs new file mode 100644 index 000000000..f389ebf63 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ConformanceMorpherFactory.cs @@ -0,0 +1,22 @@ +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Builds the oracle for a fixture. The fixture's morpher: block is a +/// required argument, so no caller can run a fixture that overrides a setting at the defaults. +/// +internal static class ConformanceMorpherFactory +{ + internal static Morpher Create(Language language, MorpherSettings settings) + { + var morpher = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + settings.ApplyTo(morpher); + return morpher; + } + + internal static Morpher CreateTracing(Language language, MorpherSettings settings) + { + var morpher = new Morpher(new TraceManager { IsTracing = true }, language, maxDegreeOfParallelism: 1); + settings.ApplyTo(morpher); + return morpher; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs new file mode 100644 index 000000000..0bdd76353 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/CoverageReport.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Coverage and dead-rule measurement over the fixtures. +/// +/// blocked_by is asserted, not verified. A rule named in an expect_fail word's +/// blocked_by is credited as exercised, but nothing checks the trace to confirm that rule +/// actually blocked the word. The only check is that every label is a rule id declared in that +/// fixture's grammar (), which catches typos and stale ids, not +/// a label naming a real rule that never ran. Verifying labels against the parse trace is tracked +/// in sillsdev/machine#552. +/// +public static class CoverageReport +{ + public class DeadRule + { + public string FixtureId = ""; + public string Language = ""; + public string RuleId = ""; + } + + public class UnknownBlockedBy + { + public string FixtureId = ""; + public string Word = ""; + public string Label = ""; + } + + public class CoverageResult + { + public List DeadRules { get; } = new(); + + /// blocked_by labels that are not a rule id in their fixture's grammar. + public List UnknownBlockedBy { get; } = new(); + + /// Every construct string any word exercises (the distinct construct column + /// of coverage.csv). Diffed against constructs.txt by the caller to report absolute + /// construct coverage. + public HashSet CoveredConstructs { get; } = new(StringComparer.Ordinal); + } + + public static List LoadConstructChecklist(string path) + { + var constructs = new List(); + foreach (string rawLine in File.ReadAllLines(path)) + { + string line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith("#", StringComparison.Ordinal)) + continue; + constructs.Add(line); + } + return constructs; + } + + /// Reads dead-rules-allowed.tsv: one fixture-id TAB rule-id TAB reason row + /// per rule a fixture declares on purpose without any word exercising it. The reason column is + /// required so that no rule is allowed without saying why. + public static Dictionary<(string FixtureId, string RuleId), string> LoadDeadRuleAllowlist(string path) + { + var allowed = new Dictionary<(string, string), string>(); + int lineNumber = 0; + foreach (string rawLine in File.ReadAllLines(path)) + { + lineNumber++; + if (rawLine.Trim().Length == 0 || rawLine.StartsWith("#", StringComparison.Ordinal)) + continue; + string[] columns = rawLine.Split('\t'); + if (columns.Length != 3 || columns.Any(c => c.Trim().Length == 0)) + throw new FormatException($"{path}:{lineNumber}: expected fixture, rule and reason separated by tabs"); + if (!allowed.TryAdd((columns[0].Trim(), columns[1].Trim()), columns[2].Trim())) + throw new FormatException($"{path}:{lineNumber}: duplicate entry"); + } + return allowed; + } + + /// Splits the measured dead rules against the allowlist. Unexpected are dead rules + /// nobody has accepted; Stale are accepted rules that some word now exercises, whose entry + /// must go so the list cannot drift into accepting rules that are no longer dead. + public static (List Unexpected, List<(string FixtureId, string RuleId)> Stale) CheckDeadRules( + IReadOnlyCollection deadRules, + IReadOnlyDictionary<(string FixtureId, string RuleId), string> allowed + ) + { + List unexpected = deadRules.Where(d => !allowed.ContainsKey((d.FixtureId, d.RuleId))).ToList(); + var dead = new HashSet<(string, string)>(deadRules.Select(d => (d.FixtureId, d.RuleId))); + List<(string, string)> stale = allowed + .Keys.Where(k => !dead.Contains(k)) + .OrderBy(k => k.FixtureId, StringComparer.Ordinal) + .ThenBy(k => k.RuleId, StringComparer.Ordinal) + .ToList(); + return (unexpected, stale); + } + + /// Returns every blocked_by label, on an expect_fail or expect_skip word, that is not + /// in (the fixture grammar's declared rule ids). Ordinal comparison. + public static List FindUnknownBlockedBy( + string fixtureId, + WordsYaml words, + IReadOnlyCollection ruleIds + ) + { + var known = new HashSet(ruleIds, StringComparer.Ordinal); + var unknown = new List(); + foreach (WordEntry word in words.Words) + { + if (!word.ExpectFail && !word.ExpectSkip) + continue; + foreach (string label in word.BlockedBy) + { + if (!known.Contains(label)) + { + unknown.Add( + new UnknownBlockedBy + { + FixtureId = fixtureId, + Word = word.Word, + Label = label, + } + ); + } + } + } + return unknown; + } + + public static CoverageResult WriteCsvs(List fixtures, string coverageCsvPath, string rulesCsvPath) + { + var result = new CoverageResult(); + var coverageRows = new List<(string language, string word, string signature, string construct)>(); + var ruleRows = new List<(string language, string ruleId, string words)>(); + + foreach (Fixture fixture in fixtures) + { + string language = fixture.Words.Language; + GrammarRuleIndex ruleIndex = GrammarRuleIndex.Load(fixture.GrammarPath); + result.UnknownBlockedBy.AddRange(FindUnknownBlockedBy(fixture.Id, fixture.Words, ruleIndex.AllRuleIds)); + + // rule id -> exercising word tokens ("word" for a positive rules: hit, "!word" for a + // blocked_by attribution on an expect_fail word). + var exercisingWords = new Dictionary>(StringComparer.Ordinal); + void AddExercise(string ruleId, string token) + { + if (!exercisingWords.TryGetValue(ruleId, out List list)) + { + list = new List(); + exercisingWords[ruleId] = list; + } + list.Add(token); + } + + foreach (WordEntry word in fixture.Words.Words) + { + if (word.ExpectFail || word.ExpectSkip) + { + foreach (string construct in word.Exercises) + { + coverageRows.Add((language, word.Word, "", construct)); + result.CoveredConstructs.Add(construct); + } + foreach (string ruleId in word.BlockedBy) + AddExercise(ruleId, "!" + word.Word); + } + else + { + foreach (ParseEntry parse in word.Parses) + { + IEnumerable constructs = parse + .Exercises.Concat(word.Exercises) + .Distinct(StringComparer.Ordinal); + foreach (string construct in constructs) + { + coverageRows.Add((language, word.Word, parse.Signature, construct)); + result.CoveredConstructs.Add(construct); + } + foreach (string ruleId in parse.Rules) + AddExercise(ruleId, word.Word); + } + } + } + + foreach ( + string ruleId in ruleIndex + .AllRuleIds.Distinct(StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + ) + { + if (exercisingWords.TryGetValue(ruleId, out List words)) + { + ruleRows.Add((language, ruleId, string.Join(";", words.Distinct(StringComparer.Ordinal)))); + } + else + { + ruleRows.Add((language, ruleId, "")); + result.DeadRules.Add( + new DeadRule + { + FixtureId = fixture.Id, + Language = language, + RuleId = ruleId, + } + ); + } + } + } + + WriteCsv( + coverageCsvPath, + new[] { "language", "word", "signature", "construct" }, + coverageRows.Select(r => new[] { r.language, r.word, r.signature, r.construct }) + ); + WriteCsv( + rulesCsvPath, + new[] { "language", "rule_id", "words" }, + ruleRows.Select(r => new[] { r.language, r.ruleId, r.words }) + ); + + return result; + } + + private static void WriteCsv(string path, string[] header, IEnumerable rows) + { + // Always \n, so the checked-in CSVs compare clean on every platform (local_check.sh diffs them). + var sb = new StringBuilder(); + sb.Append(string.Join(",", header.Select(CsvField))).Append('\n'); + foreach (string[] row in rows) + sb.Append(string.Join(",", row.Select(CsvField))).Append('\n'); + File.WriteAllText(path, sb.ToString()); + } + + private static string CsvField(string value) + { + value ??= ""; + if (value.IndexOfAny(new[] { ',', '"', '\n', '\r' }) >= 0) + return "\"" + value.Replace("\"", "\"\"") + "\""; + return value; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs new file mode 100644 index 000000000..a568f325e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Diff.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Per-word outcome of diffing one fixture's actual output against its expected.tsv. +public class WordResult +{ + public string Word = ""; + public bool Passed; + public string Detail = ""; +} + +public class FixtureDiff +{ + public List WordResults { get; } = new(); + public bool AllPassed => WordResults.All(w => w.Passed); +} + +public static class Diff +{ + public static FixtureDiff Compare(List expected, List actual) + { + var diff = new FixtureDiff(); + Dictionary> expByWord = GroupByWord(expected); + Dictionary> actByWord = GroupByWord(actual); + + IEnumerable allWords = expByWord + .Keys.Union(actByWord.Keys, StringComparer.Ordinal) + .OrderBy(w => w, StringComparer.Ordinal); + + foreach (string word in allWords) + { + bool hasExpected = expByWord.TryGetValue(word, out List expRows); + bool hasActual = actByWord.TryGetValue(word, out List actRows); + + if (!hasExpected) + { + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = false, + Detail = "present in adapter output but not in expected.tsv", + } + ); + continue; + } + if (!hasActual) + { + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = false, + Detail = "missing from adapter/self-check output", + } + ); + continue; + } + + List expCanon = Canonicalize(expRows); + List actCanon = Canonicalize(actRows); + bool passed = expCanon.SequenceEqual(actCanon, StringComparer.Ordinal); + diff.WordResults.Add( + new WordResult + { + Word = word, + Passed = passed, + Detail = passed + ? "ok" + : $"expected [{string.Join(" | ", expCanon)}] got [{string.Join(" | ", actCanon)}]", + } + ); + } + + return diff; + } + + private static Dictionary> GroupByWord(List rows) + { + var byWord = new Dictionary>(); + foreach (TsvRow row in rows) + { + if (!byWord.TryGetValue(row.Word, out List list)) + { + list = new List(); + byWord[row.Word] = list; + } + list.Add(row); + } + return byWord; + } + + // One canonical "status|sorted-signature-entries" string per row, sorted across the word's + // rows (usually just one) so occurrence order/duplication is handled as a multiset. + private static List Canonicalize(List rows) + { + return rows.Select(r => r.Status + "|" + string.Join(";", SignatureTsv.SplitComparableSignature(r.Signature))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs new file mode 100644 index 000000000..604d3941c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EngineCrashException.cs @@ -0,0 +1,21 @@ +using System; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Thrown by an when the ENGINE UNDER TEST itself failed while loading a +/// grammar or parsing a word -- a genuine engine crash, as opposed to a harness-level failure +/// (the adapter process couldn't be started, timed out, or exited 0 without producing an output +/// file; see ). treats this exception specially: a +/// fixture whose manifest declares expectCrash: true (see ) +/// PASSes when this is thrown instead of the normal signature diff; any other fixture FAILs, +/// showing the carried diagnostics. +/// +public class EngineCrashException : Exception +{ + public EngineCrashException(string message) + : base(message) { } + + public EngineCrashException(string message, Exception innerException) + : base(message, innerException) { } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets new file mode 100644 index 000000000..7909286e2 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/EnsureConformanceSuite.targets @@ -0,0 +1,254 @@ + + + + + + + + + + + + + + + + > git = (workingDirectory, arguments) => + { + var startInfo = new ProcessStartInfo("git", arguments) + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.EnvironmentVariables["GIT_TERMINAL_PROMPT"] = "0"; + startInfo.EnvironmentVariables["GIT_OPTIONAL_LOCKS"] = "0"; + startInfo.EnvironmentVariables.Remove("GIT_DIR"); + startInfo.EnvironmentVariables.Remove("GIT_WORK_TREE"); + startInfo.EnvironmentVariables.Remove("GIT_INDEX_FILE"); + try + { + using (Process process = Process.Start(startInfo)) + { + var output = new StringBuilder(); + var error = new StringBuilder(); + process.OutputDataReceived += (sender, e) => { if (e.Data != null) lock (output) output.AppendLine(e.Data); }; + process.ErrorDataReceived += (sender, e) => { if (e.Data != null) lock (error) error.AppendLine(e.Data); }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + int limit = (int)Math.Max(10000, (deadline - DateTime.UtcNow).TotalMilliseconds); + if (!process.WaitForExit(limit)) + { + try + { + var killTree = typeof(Process).GetMethod("Kill", new[] { typeof(bool) }); + if (killTree != null) killTree.Invoke(process, new object[] { true }); + else process.Kill(); + } + catch (Exception) { } + process.WaitForExit(5000); + return new KeyValuePair(-1, "git did not finish in time: git " + arguments); + } + process.WaitForExit(); + return new KeyValuePair(process.ExitCode, process.ExitCode == 0 ? output.ToString() : error.ToString() + output.ToString()); + } + } + catch (Exception exception) + { + return new KeyValuePair(-1, "git could not be run: " + exception.Message); + } + }; + + Action deleteQuietly = path => + { + // Git writes its object files read-only, and Windows will not delete those. + try + { + foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + File.SetAttributes(file, FileAttributes.Normal); + Directory.Delete(path, true); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + }; + + try + { + // A temporary clone left by an interrupted build is never reused; remove the old ones. + if (Directory.Exists(parent)) + { + foreach (string stale in Directory.GetDirectories(parent, name + ".tmp-*")) + { + if (DateTime.UtcNow - Directory.GetCreationTimeUtc(stale) > TimeSpan.FromHours(1)) + deleteQuietly(stale); + } + } + + bool present = Directory.Exists(suite) && Directory.EnumerateFileSystemEntries(suite).Any(); + if (present) + { + if (!Directory.Exists(Path.Combine(suite, ".git")) && !File.Exists(Path.Combine(suite, ".git"))) + { + Log.LogError("conformance: " + suite + " exists but is not a git clone. Move it aside and build again, or set HC_CONFORMANCE_SUITE= to use it as it is."); + return false; + } + + KeyValuePair head = git(suite, "rev-parse HEAD"); + if (head.Key != 0) + { + Log.LogWarning("conformance: could not read the commit of " + suite + " (" + head.Value.Trim() + "). Run these two commands:" + movePin); + return true; + } + + KeyValuePair status = git(suite, "status --porcelain"); + string[] changed = status.Key == 0 + ? status.Value.Split('\n').Select(line => line.TrimEnd('\r')).Where(line => line.Length > 3).Select(line => line.Substring(3)).ToArray() + : new string[0]; + + if (head.Value.Trim() == pin) + { + // A suite author may be testing a change, so this is a warning and the build goes on. + if (changed.Length > 0) + Log.LogWarning("conformance: " + suite + " is at the pinned commit but has " + changed.Length + " local change(s): " + string.Join(", ", changed.Take(5)) + (changed.Length > 5 ? ", ..." : "") + ". The tests and the runner use what is on disk. Discard them unless you are testing a suite change."); + return true; + } + + if (changed.Length > 0) + { + Log.LogError("conformance: " + suite + " is at " + head.Value.Trim() + ", not the pinned " + pin + ", and has uncommitted changes (" + changed[0] + "), so it was left alone. Commit or discard them and build again, or set HC_CONFORMANCE_SUITE=" + suite + " to use it as it is."); + return false; + } + + if (git(suite, "cat-file -e " + pin + "^{commit}").Key != 0) + { + KeyValuePair fetch = git(suite, stalled + "fetch --quiet origin"); + if (fetch.Key != 0) + { + Log.LogWarning("conformance: " + suite + " is not at the pinned commit and fetching it failed (" + fetch.Value.Trim() + "). Run these two commands:" + movePin); + return true; + } + } + + KeyValuePair checkout = git(suite, stalled + "-c advice.detachedHead=false checkout --quiet --detach " + pin); + if (checkout.Key != 0) + Log.LogWarning("conformance: could not move " + suite + " to " + pin + " (" + checkout.Value.Trim() + "). Run this command: git -C " + name + " checkout --detach " + pin); + else + Log.LogMessage(MessageImportance.High, "conformance: moved " + suite + " from " + head.Value.Trim() + " to the pinned " + pin + "."); + return true; + } + + string temporary = Path.Combine(parent, name + ".tmp-" + Guid.NewGuid().ToString("N")); + Log.LogMessage(MessageImportance.High, "conformance: cloning " + SuiteUrl + " at " + pin + " into " + suite + "."); + KeyValuePair clone = git(parent, stalled + "clone --quiet --filter=blob:none --no-checkout " + SuiteUrl + " \"" + temporary + "\""); + if (clone.Key == 0) + clone = git(temporary, stalled + "-c advice.detachedHead=false checkout --quiet --detach " + pin); + if (clone.Key != 0) + { + deleteQuietly(temporary); + Log.LogWarning("conformance: could not clone the suite (" + clone.Value.Trim() + "). Run these two commands:" + clonePin); + return true; + } + + // Defender or the search indexer can hold a handle in the new tree for a moment, so retry the rename. + string constructs = Path.Combine(suite, "constructs.txt"); + string renameError = ""; + bool moved = false; + for (int attempt = 0; attempt < 5 && !moved; attempt++) + { + try + { + if (Directory.Exists(suite) && !Directory.EnumerateFileSystemEntries(suite).Any()) + Directory.Delete(suite); + Directory.Move(temporary, suite); + moved = true; + } + catch (Exception exception) when (exception is IOException || exception is UnauthorizedAccessException) + { + renameError = exception.Message; + if (File.Exists(constructs)) + break; + System.Threading.Thread.Sleep(500); + } + } + + if (!moved) + { + if (File.Exists(constructs)) + { + // A parallel build finished first; its clone stands if it is at the pin. + deleteQuietly(temporary); + KeyValuePair raced = git(suite, "rev-parse HEAD"); + if (raced.Key != 0 || raced.Value.Trim() != pin) + Log.LogWarning("conformance: " + suite + " appeared during the build and is not at " + pin + ". Run these two commands:" + movePin); + } + else + { + Log.LogWarning("conformance: cloned the suite but could not rename " + temporary + " to " + suite + " (" + renameError + "). The clone is complete: rename that directory to " + name + ", or build again."); + } + } + } + catch (Exception exception) + { + Log.LogWarning("conformance: could not make " + suite + " the suite at " + pin + " (" + exception.GetType().Name + ": " + exception.Message + "). Run these two commands:" + clonePin); + } + ]]> + + + + + + + + + diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs new file mode 100644 index 000000000..913cad0b7 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Fixture.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// One fixture directory: exactly grammar.xml + words.yaml, under +/// conformance/languages/<name>/ or conformance/edge-cases/<name>/. +/// +public class Fixture(string id, string directory, WordsYaml words) +{ + public string Id { get; } = id; + public string Directory { get; } = directory; + public WordsYaml Words { get; } = words; + + public string GrammarPath => Path.Combine(Directory, "grammar.xml"); + + /// The only directories under a fixtures root that hold fixtures. + public static readonly string[] Categories = { "languages", "edge-cases" }; + + public static List DiscoverAll(string fixturesRoot) + { + var fixtures = new List(); + foreach (string category in Categories) + { + string categoryRoot = Path.Combine(fixturesRoot, category); + if (!System.IO.Directory.Exists(categoryRoot)) + continue; + + foreach (string dir in System.IO.Directory.EnumerateDirectories(categoryRoot)) + { + string grammarPath = Path.Combine(dir, "grammar.xml"); + string wordsYamlPath = Path.Combine(dir, "words.yaml"); + if (!File.Exists(grammarPath) || !File.Exists(wordsYamlPath)) + continue; + + string name = Path.GetFileName(dir); + string id = $"{category}/{name}"; + WordsYaml words = WordsYamlLoader.Load(wordsYamlPath); + fixtures.Add(new Fixture(id, dir, words)); + } + } + fixtures.Sort((a, b) => string.CompareOrdinal(a.Id, b.Id)); + return fixtures; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs new file mode 100644 index 000000000..4ae50f35f --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureManifest.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// In-memory adapter-mode fixture descriptor, built by from a +/// fixture's grammar.xml + words.yaml. It is never read from or written to disk. +/// +public class FixtureManifest +{ + public string Id { get; set; } = ""; + + public string Category { get; set; } = "single-feature"; + + public List Requires { get; set; } = new(); + + /// The fixture's morpher: block; an engine runs this fixture only if it declares + /// , and must apply every value. + public MorpherSettings Morpher { get; set; } = new(); + + public bool ExpectCrash { get; set; } + + public FixtureBudget Budget { get; set; } + + public string Provenance { get; set; } = ""; +} + +/// +/// A pathological fixture's complexity budget: a wall-clock bound only, with no step-count ceiling. +/// Absent/null for non-pathological fixtures. +/// +public class FixtureBudget +{ + public long WallClockMs { get; set; } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs new file mode 100644 index 000000000..04e81e10c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/FixtureMaterializer.cs @@ -0,0 +1,126 @@ +using System; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Materializes a fixture (grammar.xml + words.yaml) into the on-disk adapter shape +/// (grammar.xml + words.txt + expected.tsv) in a temp directory, plus an in-memory +/// , so it can be run through + +/// for adapter mode without either needing to know about the +/// words.yaml format. This is the only code path that touches AdapterEngine/MaterializedRunner. +/// +public static class FixtureMaterializer +{ + /// + /// Builds the temp directory and returns a pointing at it. + /// Caller owns cleanup (delete the returned fixture's + /// when done). + /// + public static MaterializedFixture Materialize(Fixture fixture) + { + return Materialize(fixture, new WordRun("default", fixture.Words.Morpher, fixture.Words.Words)); + } + + public static MaterializedFixture Materialize(Fixture fixture, WordRun run) + { + var adapterWords = run.Morpher.GuessRoots.HasValue + ? run.Words + : run.Words.Where(w => w.Parses.All(p => !p.Guess)).ToList(); + if (run.Name != "default" && adapterWords.Count == 0) + { + throw new NamedVariantHasNoAdapterVisibleWordsException( + fixture.Id, + run.Name, + "no adapter-visible words remain after legacy guess filtering; set morpher.GuessRoots explicitly " + + "if the variant is intended to include guessed-word rows" + ); + } + + string tempDir = Path.Combine(Path.GetTempPath(), "hc-conformance-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + File.Copy(fixture.GrammarPath, Path.Combine(tempDir, "grammar.xml")); + + string wordsPath = Path.Combine(tempDir, "words.txt"); + File.WriteAllLines(wordsPath, adapterWords.Select(w => w.Word)); + + string expectedPath = Path.Combine(tempDir, "expected.tsv"); + using (var writer = new StreamWriter(expectedPath)) + { + if (fixture.Words.ExpectCrash) + { + WordEntry first = fixture.Words.Words[0]; + writer.WriteLine($"0\t{first.Word}\tSTARTED"); + } + else + { + int i = 0; + foreach (WordEntry word in adapterWords) + { + string signature = BuildExpectedSignature(word); + // expect_skip words carry status "SKIPPED" (the oracle throws InvalidShapeException), + // matching the expected.tsv row shape "N word 0 SKIPPED -"; everything else is "ok". + string status = word.ExpectSkip ? "SKIPPED" : "ok"; + writer.WriteLine($"{i}\t{word.Word}\t0\t{status}\t{signature}"); + i++; + } + } + } + + File.WriteAllText( + Path.Combine(tempDir, MaterializedFixture.MorpherFileName), + System.Text.Json.JsonSerializer.Serialize(run.Morpher.ToDictionary()) + ); + + var manifest = new FixtureManifest + { + Id = fixture.Id, + Category = fixture.Words.BudgetMs.HasValue ? "pathological" : "single-feature", + Requires = fixture.Words.Requires, + Morpher = run.Morpher, + ExpectCrash = fixture.Words.ExpectCrash, + Budget = fixture.Words.BudgetMs.HasValue + ? new FixtureBudget { WallClockMs = fixture.Words.BudgetMs.Value } + : null, + Provenance = string.Join( + "; ", + run.Words.Where(w => !string.IsNullOrEmpty(w.Provenance)).Select(w => w.Provenance) + ), + }; + + bool requiresMorpherSettings = RequiresDerivation + .Derive(fixture.GrammarPath, fixture.Words.Runs.Select(candidate => candidate.Morpher)) + .Contains(MorpherSettings.Capability, StringComparer.Ordinal); + return new MaterializedFixture( + tempDir, + manifest, + run.Name == "default" ? null : run.Name, + requiresMorpherSettings + ); + } + + public static void Cleanup(MaterializedFixture materialized) + { + try + { + if (Directory.Exists(materialized.Directory)) + Directory.Delete(materialized.Directory, recursive: true); + } + catch (IOException) { } + } + + /// The signature string materialized into expected.tsv column 5: "-" for a word with zero expected + /// parses (expect_fail), else the ";"-joined, ordinally-sorted set of parses[].signature -- the same shape + /// BatchCommand.BuildSignature itself produces. + public static string BuildExpectedSignature(WordEntry word) + { + if (word.ExpectFail || word.ExpectSkip || word.Parses.Count == 0) + return "-"; + return string.Join(";", word.Parses.Select(p => p.Signature).OrderBy(s => s, StringComparer.Ordinal)); + } +} + +internal sealed class NamedVariantHasNoAdapterVisibleWordsException(string fixtureId, string variantName, string reason) + : InvalidOperationException($"fixture '{fixtureId}' variant '{variantName}' {reason}.") { } diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs new file mode 100644 index 000000000..05e9efd60 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarRuleIndex.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml; +using System.Xml.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Independently re-parses a fixture's grammar.xml (not via , +/// whose runtime rule objects lose the XML id attribute for several rule kinds -- see below) +/// to answer two questions the harness needs for the "rules:" coverage dimension: (1) every rule id +/// the grammar declares (for dead-rule detection), and (2) how to translate a *runtime* rule +/// identity (what or a node's +/// Source actually expose at parse time) back to that XML id attribute, so a traced +/// rule application can be compared against a words.yaml "rules:" entry. +/// +/// Why this indirection is needed (see XmlLanguageLoader.TryLoadAffixProcessRule / +/// TryLoadCompoundingRule / LoadRewriteRule / LoadMetathesisRule): the XML +/// id attribute on a MorphologicalRule/RealizationalRule element is used only +/// as the loader's own lookup key while building the graph -- the *runtime* +/// object's is loaded from the separate <MorphemeId> child +/// element instead (the short code that appears in a parse's morph-chain signature, e.g. "PL"). +/// A CompoundingRule, PhonologicalRule, and MetathesisRule element has no +/// MorphemeId at all; their runtime objects expose only (from the +/// element's <Name> child). None of the four runtime rule kinds retains the XML +/// id attribute itself anywhere the harness can read it back after loading. This class +/// rebuilds that lost mapping straight from the XML. +/// +public class GrammarRuleIndex +{ + /// Every rule id declared by the grammar (MorphologicalRule/RealizationalRule/CompoundingRule/ + /// PhonologicalRule/MetathesisRule ids, plus the co-occurrence-rule pseudo-ids below), in document + /// order. + public List AllRuleIds { get; } = new(); + + private readonly Dictionary _ruleIdByMorphemeId = new(StringComparer.Ordinal); + + // text -> XML id attribute, for CompoundingRule/PhonologicalRule/MetathesisRule: the only + // identity these three retain at runtime is IHCRule.Name. + private readonly Dictionary _ruleIdByRuleName = new(StringComparer.Ordinal); + + public HashSet CoOccurrencePseudoIds { get; } = new(StringComparer.Ordinal); + + public static GrammarRuleIndex Load(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + var index = new GrammarRuleIndex(); + foreach (XElement el in doc.Descendants()) + { + switch (el.Name.LocalName) + { + case "MorphologicalRule": + case "RealizationalRule": + { + string id = (string)el.Attribute("id"); + string morphemeId = (string)el.Element("MorphemeId"); + if (id == null) + continue; + index.AllRuleIds.Add(id); + if (morphemeId != null) + { + AddUnique(index._ruleIdByMorphemeId, morphemeId, id, grammarPath, "MorphemeId"); + } + else + { + string name = (string)el.Element("Name"); + if (name != null) + AddUnique(index._ruleIdByRuleName, name, id, grammarPath, "Name"); + } + break; + } + case "CompoundingRule": + case "PhonologicalRule": + case "MetathesisRule": + { + string id = (string)el.Attribute("id"); + string name = (string)el.Element("Name"); + if (id == null) + continue; + index.AllRuleIds.Add(id); + if (name != null) + AddUnique(index._ruleIdByRuleName, name, id, grammarPath, "Name"); + break; + } + case "MorphemeCoOccurrenceRule": + case "AllomorphCoOccurrenceRule": + { + string primary = (string)(el.Attribute("primaryMorpheme") ?? el.Attribute("primaryAllomorph")); + if (primary == null) + continue; + index.AllRuleIds.Add(primary); + index.CoOccurrencePseudoIds.Add(primary); + break; + } + } + } + return index; + } + + /// Records under , throwing if a different rule already + /// claimed that key: a fired rule is resolved by key, so a shared key would credit the wrong rule. + private static void AddUnique( + Dictionary map, + string key, + string id, + string grammarPath, + string keyKind + ) + { + if (map.TryGetValue(key, out string existing) && !string.Equals(existing, id, StringComparison.Ordinal)) + { + throw new InvalidDataException( + $"{grammarPath}: rules '{existing}' and '{id}' share the {keyKind} '{key}', so a fired rule cannot be attributed" + ); + } + map[key] = id; + } + + /// + /// Translates a fired (from , + /// or a RealizationalAffixProcessRule taken from a MorphologicalRuleSynthesis trace node -- see + /// ) back to its grammar.xml rule id. Returns null (rather than + /// throwing) for a rule kind this index cannot attribute. + /// + public string ResolveMorphologicalRuleId(IMorphologicalRule rule) + { + if ( + rule is Morpheme morpheme + && morpheme.Id != null + && _ruleIdByMorphemeId.TryGetValue(morpheme.Id, out string byMorphemeId) + ) + { + return byMorphemeId; + } + if (rule.Name != null && _ruleIdByRuleName.TryGetValue(rule.Name, out string byName)) + return byName; + return null; + } + + /// Translates a fired phonological rule (from a trace node's Source) back to its grammar.xml + /// rule id, by IHCRule.Name -- see the class doc comment. + public string ResolvePhonologicalRuleId(IPhonologicalRule rule) + { + return rule.Name != null && _ruleIdByRuleName.TryGetValue(rule.Name, out string id) ? id : null; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs new file mode 100644 index 000000000..bacd3dfbc --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/GrammarValidation.cs @@ -0,0 +1,75 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Validates a fixture grammar against the pinned DTD with no external reach. +/// The only admitted external subset is the exact system identifier +/// HermitCrabInput.dtd, remapped to the suite's canonical DTD whatever directory the +/// fixture lives in. Network access and filesystem fallback are refused rather than attempted. +public static class GrammarValidation +{ + public const string AdmittedSystemId = "HermitCrabInput.dtd"; + + /// Validates , returning every validation message. + public static IReadOnlyList Validate(string grammarPath, string dtdPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(grammarPath); + ArgumentException.ThrowIfNullOrWhiteSpace(dtdPath); + + var messages = new List(); + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Parse, + ValidationType = ValidationType.DTD, + XmlResolver = new PinnedDtdResolver(Path.GetFullPath(dtdPath)), + }; + settings.ValidationEventHandler += (_, args) => messages.Add($"{args.Severity}: {args.Message}"); + + // The document is opened here so the resolver is consulted only for the DTD, never for the + // grammar's own URI. + using var grammar = new FileStream(grammarPath, FileMode.Open, FileAccess.Read, FileShare.Read); + using XmlReader reader = XmlReader.Create( + grammar, + settings, + new Uri(Path.GetFullPath(grammarPath)).AbsoluteUri + ); + while (reader.Read()) + { + // Validation is reported through the event handler; the read loop only drives it. + } + + return messages; + } + + private sealed class PinnedDtdResolver(string dtdPath) : XmlResolver + { + private readonly string _dtdPath = dtdPath; + + // Called for the document's own base URI as well as for external subsets, so admission is + // decided in GetEntity, which is where content would actually be opened. + public override Uri ResolveUri(Uri? baseUri, string? relativeUri) => + string.Equals(relativeUri, AdmittedSystemId, StringComparison.Ordinal) + ? new Uri(_dtdPath) + : base.ResolveUri(baseUri, relativeUri); + + public override object GetEntity(Uri absoluteUri, string? role, Type? typeOfObjectToReturn) + { + ArgumentNullException.ThrowIfNull(absoluteUri); + if ( + !absoluteUri.IsFile + || !string.Equals(Path.GetFullPath(absoluteUri.LocalPath), _dtdPath, StringComparison.OrdinalIgnoreCase) + ) + { + throw new XmlException( + $"External entity '{absoluteUri}' is not admitted; only '{AdmittedSystemId}' may be referenced." + ); + } + + return new FileStream(_dtdPath, FileMode.Open, FileAccess.Read, FileShare.Read); + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs new file mode 100644 index 000000000..5a603ae94 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/IEngine.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public interface IEngine +{ + /// A short, human-readable name for reporting (e.g. "adapter"). + string Name { get; } + + /// + /// The capability set this engine declares support for; an adapter's set comes from the + /// --capabilities flag. + /// + IReadOnlySet Capabilities { get; } + + /// Runs every word in the fixture's words.txt, returning the resulting TSV rows. + List Run(MaterializedFixture fixture); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs new file mode 100644 index 000000000..4b637fc10 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedFixture.cs @@ -0,0 +1,38 @@ +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// A fixture materialized onto disk in the on-disk adapter shape (grammar.xml + words.txt + +/// expected.tsv + an in-memory ), so it can be handed to +/// and (an external adapter +/// process). Built by from a . +/// +public class MaterializedFixture( + string directory, + FixtureManifest manifest, + string variantName = null, + bool? requiresMorpherSettings = null +) +{ + public string Id { get; } = variantName == null ? manifest.Id : $"{manifest.Id} [variant {variantName}]"; + public string Directory { get; } = directory; + public FixtureManifest Manifest { get; } = manifest; + + /// + /// Fixture-level capability requirement. This can be true for the default run even when its own + /// settings are empty, because one of the fixture's named variants declares settings. + /// + public bool RequiresMorpherSettings { get; } = requiresMorpherSettings ?? !manifest.Morpher.IsEmpty; + + public string GrammarPath => Path.Combine(Directory, "grammar.xml"); + public string WordsPath => Path.Combine(Directory, "words.txt"); + public string ExpectedPath => Path.Combine(Directory, "expected.tsv"); + + /// + /// The JSON object of this run's morpher: block, keyed by FieldWorks element name; {} when absent. + /// + public const string MorpherFileName = "morpher.json"; + + public string MorpherPath => Path.Combine(Directory, MorpherFileName); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs new file mode 100644 index 000000000..a76d7d706 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MaterializedRunner.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public enum FixtureOutcome +{ + Passed, + Failed, + Skipped, +} + +public class FixtureResult +{ + public string FixtureId = ""; + public FixtureOutcome Outcome; + public string Reason = ""; + public long ElapsedMs; + public List WordResults = new(); +} + +public class MaterializedRunReport +{ + public List Results { get; } = new(); + public int ExcludedPathologicalCount { get; set; } + + public int Passed => Results.Count(r => r.Outcome == FixtureOutcome.Passed); + public int Failed => Results.Count(r => r.Outcome == FixtureOutcome.Failed); + public int Skipped => Results.Count(r => r.Outcome == FixtureOutcome.Skipped); +} + +/// +/// Runs a list of s through an , applying +/// category exclusion (pathological) and capability filtering (manifest "requires" vs. the engine's +/// declared capabilities), and diffs results against each fixture's expected.tsv. Called by +/// against fixtures writes to disk. +/// +public static class MaterializedRunner +{ + public static MaterializedRunReport Run( + List fixtures, + IEngine engine, + bool includePathological + ) + { + var report = new MaterializedRunReport(); + + foreach (MaterializedFixture fixture in fixtures) + { + var result = new FixtureResult { FixtureId = fixture.Id }; + if (IsExcludedPathological(fixture, includePathological)) + { + report.ExcludedPathologicalCount++; + continue; + } + + List derivedRequires = DerivedRequires(fixture); + if (!RequiresMatchGrammar(fixture, derivedRequires)) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"manifest error: requires [{string.Join(",", fixture.Manifest.Requires)}] does not match " + + $"grammar-derived [{string.Join(",", derivedRequires)}]"; + report.Results.Add(result); + continue; + } + + List missingCapabilities = MissingCapabilities(fixture, engine); + if (missingCapabilities.Count > 0) + { + result.Outcome = FixtureOutcome.Skipped; + result.Reason = + $"requires [{string.Join(",", fixture.Manifest.Requires)}], " + + $"engine declares [{string.Join(",", engine.Capabilities.OrderBy(c => c, StringComparer.Ordinal))}]; " + + $"missing [{string.Join(",", missingCapabilities)}]"; + report.Results.Add(result); + continue; + } + + // Read once, before the try, so a missing expected.tsv cannot throw from inside a catch block. + List expected = SignatureTsv.ReadFile(fixture.ExpectedPath); + + var sw = Stopwatch.StartNew(); + try + { + List actual = RunWithBudget(engine, fixture); + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Manifest.ExpectCrash) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = "expected crash, engine produced output"; + } + else + { + FixtureDiff diff = Diff.Compare(expected, actual); + result.WordResults = diff.WordResults; + + // Enforced whenever a budget is present, regardless of category: today only + // pathological fixtures carry one, but nothing about the check is category- + // specific. + bool overBudget = + fixture.Manifest.Budget != null && result.ElapsedMs > fixture.Manifest.Budget.WallClockMs; + + if (overBudget) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"exceeded wall-clock budget ({result.ElapsedMs}ms > {fixture.Manifest.Budget.WallClockMs}ms)"; + } + else if (diff.AllPassed) + { + result.Outcome = FixtureOutcome.Passed; + } + else + { + result.Outcome = FixtureOutcome.Failed; + // Name the words. A bare count cannot be diagnosed from a CI log on a machine you cannot reach. + + WordResult[] mismatched = diff.WordResults.Where(word => !word.Passed).ToArray(); + + string named = string.Join( + "; ", + mismatched.Take(5).Select(word => $"{word.Word}: {Truncate(word.Detail)}") + ); + + result.Reason = + mismatched.Length > 5 + ? $"{mismatched.Length}/{ +diff.WordResults +.Count} word(s) mismatched: {named}; and {mismatched.Length - 5} more" + : $"{mismatched.Length}/{ +diff.WordResults +.Count} word(s) mismatched: {named}"; + } + } + } + catch (EngineCrashException ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Manifest.ExpectCrash) + { + // A misconfigured adapter exits nonzero on every fixture, so credit the crash only + // if this engine also completes an ordinary fixture this run would execute. + MaterializedFixture control = fixtures.FirstOrDefault(candidate => + !candidate.Manifest.ExpectCrash && WouldRun(candidate, engine, includePathological) + ); + bool adapterIsFunctional = control == null || EngineCanRunControl(engine, control); + if (adapterIsFunctional) + { + result.Outcome = FixtureOutcome.Passed; + result.Reason = $"engine crashed as expected (manifest expectCrash: true): {ex.Message}"; + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"engine crashed, but also failed the control fixture '{control.Id}' -- " + + $"treating as a harness-level problem, not a crash match: {ex.Message}"; + } + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"engine crashed: {ex.Message}"; + } + } + catch (TimeoutException ex) when (fixture.Manifest.Budget != null) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"exceeded budget {fixture.Manifest.Budget.WallClockMs}ms: {ex.Message}"; + } + catch (Exception ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + // Any other exception is a plain harness error (bad --adapter template, process + // couldn't start, timed out, etc.) -- never a pass, regardless of expectCrash. + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"harness error: {ex.Message}"; + } + + report.Results.Add(result); + } + + return report; + } + + /// + /// Runs the engine for one fixture. If the manifest carries a budget, engine.Run happens on a + /// background thread and this waits up to budget + a small margin; AdapterEngine independently + /// plumbs the same budget into its own process timeout (so a subprocess adapter gets killed), + /// but an in-process engine has no external process to kill, so this watchdog is what + /// keeps a runaway engine from hanging the whole harness run. If the watchdog expires, + /// a TimeoutException is thrown and the background thread is simply abandoned -- this is a + /// short-lived CLI, so a leaked runaway thread on the rare budget-blown fixture is an acceptable + /// trade for keeping this shape simple. + /// + private static List RunWithBudget(IEngine engine, MaterializedFixture fixture) + { + FixtureBudget budget = fixture.Manifest.Budget; + if (budget == null) + return engine.Run(fixture); + + Task> task = Task.Run(() => engine.Run(fixture)); + long marginMs = Math.Max(budget.WallClockMs / 5, 1000); + bool completed; + try + { + completed = task.Wait((int)(budget.WallClockMs + marginMs)); + } + catch (AggregateException ex) when (ex.InnerException != null) + { + // Unwrap: rethrow the engine's own exception so the caller's EngineCrashException/ + // generic catch blocks see the real exception type, not Task's wrapper. + throw ex.InnerException; + } + if (!completed) + throw new TimeoutException("watchdog timed out waiting for engine.Run"); + return task.GetAwaiter().GetResult(); + } + + private static bool IsExcludedPathological(MaterializedFixture fixture, bool includePathological) => + !includePathological && string.Equals(fixture.Manifest.Category, "pathological", StringComparison.Ordinal); + + private static List DerivedRequires(MaterializedFixture fixture) => + RequiresDerivation.Derive(fixture.GrammarPath, fixture.RequiresMorpherSettings); + + private static bool RequiresMatchGrammar(MaterializedFixture fixture, List derivedRequires) => + new HashSet(derivedRequires, StringComparer.Ordinal).SetEquals( + new HashSet(fixture.Manifest.Requires, StringComparer.Ordinal) + ); + + private static List MissingCapabilities(MaterializedFixture fixture, IEngine engine) => + fixture.Manifest.Requires.Where(req => !engine.Capabilities.Contains(req)).ToList(); + + /// + /// Whether would execute this fixture rather than exclude it, report a manifest + /// error, or skip it for a missing capability. + /// + private static bool WouldRun(MaterializedFixture fixture, IEngine engine, bool includePathological) => + !IsExcludedPathological(fixture, includePathological) + && RequiresMatchGrammar(fixture, DerivedRequires(fixture)) + && MissingCapabilities(fixture, engine).Count == 0; + + /// + /// Whether this exact engine can still complete an ordinary fixture, run fresh (not read back + /// from an earlier result) so the check is independent of run order or a single-fixture run. Any + /// exception -- crash, timeout, or otherwise -- means no: only a normal return counts. + /// + private static bool EngineCanRunControl(IEngine engine, MaterializedFixture control) + { + try + { + RunWithBudget(engine, control); + return true; + } + catch + { + return false; + } + } + + /// Shortens a per-word detail so a fixture reason stays readable in a log. + /// The word detail, which may carry every expected and actual parse. + private static string Truncate(string detail) + { + const int Limit = 160; + if (string.IsNullOrEmpty(detail) || detail.Length <= Limit) + { + return detail; + } + + return detail[..Limit] + "..."; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs new file mode 100644 index 000000000..fbe202e1c --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/MorpherSettings.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// A fixture's morpher: front-matter block: host settings the run's +/// ground truth was generated under, keyed by the FieldWorks <ParserParameters><HC> +/// element names that set them (HCParser.LoadParser). Every property is null when the block +/// omits it, which means the constructor default. Only settings whose effect +/// is a parse result are accepted; anything else is a +/// error. +/// +public class MorpherSettings +{ + /// The requires capability a fixture with a non-empty block must declare. + public const string Capability = "morpher-settings"; + + /// + /// No overrides: every property at its constructor default. Named so a caller + /// that deliberately ignores a fixture's block says so. + /// + public static MorpherSettings Defaults => new(); + + /// DelReapps: , at least 0. + public int? DelReapps { get; set; } + + /// MaxRoots: , at least 1. + public int? MaxRoots { get; set; } + + /// MergeAnalyses: . + public bool? MergeAnalyses { get; set; } + + /// MakePartialsFinal: . + public bool? MakePartialsFinal { get; set; } + + /// Per-call ParseWord guessRoot flag; this is not a Morpher property. + public bool? GuessRoots { get; set; } + + public bool IsEmpty => + DelReapps == null + && MaxRoots == null + && MergeAnalyses == null + && MakePartialsFinal == null + && GuessRoots == null; + + /// + /// Resolve the per-call root-guess flag for a word, preserving legacy per-word behavior unless + /// this run explicitly supplies GuessRoots. + /// + public bool GuessRootsFor(WordEntry word) + { + return GuessRoots ?? word.Parses.Any(parse => parse.Guess); + } + + /// Sets every declared value on , leaving the rest at their defaults. + public void ApplyTo(Morpher morpher) + { + if (DelReapps is int delReapps) + morpher.DeletionReapplications = delReapps; + if (MaxRoots is int maxRoots) + morpher.MaxStemCount = maxRoots; + if (MergeAnalyses is bool mergeAnalyses) + morpher.MergeEquivalentAnalyses = mergeAnalyses; + if (MakePartialsFinal is bool makePartialsFinal) + morpher.AlwaysEnforceFinalTemplates = makePartialsFinal; + } + + /// The declared values by FieldWorks element name, for manifest.json and adapters. + public SortedDictionary ToDictionary() + { + var values = new SortedDictionary(System.StringComparer.Ordinal); + if (DelReapps is int delReapps) + values["DelReapps"] = delReapps; + if (MaxRoots is int maxRoots) + values["MaxRoots"] = maxRoots; + if (MergeAnalyses is bool mergeAnalyses) + values["MergeAnalyses"] = mergeAnalyses; + if (MakePartialsFinal is bool makePartialsFinal) + values["MakePartialsFinal"] = makePartialsFinal; + if (GuessRoots is bool guessRoots) + values["GuessRoots"] = guessRoots; + return values; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs new file mode 100644 index 000000000..921232739 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Program.cs @@ -0,0 +1,455 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +internal class Program +{ + private static int Main(string[] args) + { + string fixturesRoot = null; + string adapterTemplate = null; + string capabilitiesArg = null; + bool capabilitiesProvided = false; + bool includePathological = false; + bool coverageReport = false; + string constructsPath = null; + string coverageOutput = null; + bool propose = false; + bool checkManifest = false; + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--fixtures": + if (!TryGetNextArg(args, ref i, "--fixtures", out fixturesRoot)) + return 2; + break; + case "--adapter": + if (!TryGetNextArg(args, ref i, "--adapter", out adapterTemplate)) + return 2; + break; + case "--capabilities": + if (!TryGetNextArg(args, ref i, "--capabilities", out capabilitiesArg)) + return 2; + capabilitiesProvided = true; + break; + case "--include-pathological": + includePathological = true; + break; + case "--coverage-report": + coverageReport = true; + break; + case "--constructs": + if (!TryGetNextArg(args, ref i, "--constructs", out constructsPath)) + return 2; + break; + case "--coverage-output": + if (!TryGetNextArg(args, ref i, "--coverage-output", out coverageOutput)) + return 2; + break; + case "--propose": + propose = true; + break; + case "--check-manifest": + checkManifest = true; + break; + case "-h": + case "--help": + PrintUsage(); + return 0; + default: + Console.Error.WriteLine($"unrecognized argument: {args[i]}"); + PrintUsage(); + return 2; + } + } + + if (string.IsNullOrEmpty(fixturesRoot)) + { + Console.Error.WriteLine("--fixtures is required"); + PrintUsage(); + return 2; + } + + foreach (string warning in SuitePinCheck.Warnings(fixturesRoot)) + Console.Error.WriteLine($"warning: {warning}"); + + List fixtures; + try + { + fixtures = Fixture.DiscoverAll(fixturesRoot); + } + catch (WordsYamlException ex) + { + Console.Error.WriteLine($"words.yaml error: {ex.Message}"); + return 2; + } + Console.WriteLine($"discovered {fixtures.Count} fixture(s) under '{fixturesRoot}'"); + + if (fixtures.Count == 0) + { + Console.Error.WriteLine( + $"no fixtures discovered under '{fixturesRoot}'; if this is the conformance/ clone of the suite, " + + "run: dotnet build src/SIL.Machine.Morphology.HermitCrab.Conformance" + ); + return 2; + } + + if (checkManifest) + return RunCheckManifest(fixturesRoot); + + if (coverageReport) + { + constructsPath ??= Path.Combine(fixturesRoot, "constructs.txt"); + return RunCoverageReport(fixtures, fixturesRoot, constructsPath, coverageOutput); + } + + RunReport report; + string engineName; + IReadOnlySet engineCapabilities; + if (adapterTemplate != null) + { + var engine = new AdapterEngine(adapterTemplate, ParseCapabilities(capabilitiesArg ?? "")); + report = Runner.RunAdapter(fixtures, engine, includePathological); + engineName = engine.Name; + engineCapabilities = engine.Capabilities; + } + else + { + engineName = "self-check"; + engineCapabilities = capabilitiesProvided ? ParseCapabilities(capabilitiesArg) : Runner.DefaultCapabilities; + report = Runner.RunSelfCheck(fixtures, includePathological, engineCapabilities, propose, Console.Out); + } + PrintRunReport(report, engineName, engineCapabilities); + + bool anyRan = report.Passed > 0 || report.Failed > 0; + if (!anyRan) + { + Console.Error.WriteLine("no fixtures actually ran (all excluded or skipped) -- treating this as an error"); + return 2; + } + + return report.Failed > 0 ? 1 : 0; + } + + /// Returns the argument following , or prints usage and returns + /// false if was the last token (no bounds-check crash on a truncated + /// command line). + private static bool TryGetNextArg(string[] args, ref int i, string flag, out string value) + { + if (i + 1 >= args.Length) + { + Console.Error.WriteLine($"{flag} requires a value"); + PrintUsage(); + value = null; + return false; + } + value = args[++i]; + return true; + } + + private static HashSet ParseCapabilities(string arg) + { + return arg.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.Ordinal); + } + + private static void PrintRunReport(RunReport report, string engineName, IReadOnlySet capabilities) + { + Console.WriteLine(); + Console.WriteLine( + $"engine: {engineName}, capabilities: [{string.Join(",", capabilities.OrderBy(c => c, StringComparer.Ordinal))}]" + ); + if (report.ExcludedPathologicalCount > 0) + { + Console.WriteLine( + $"{report.ExcludedPathologicalCount} pathological run(s) (budget_ms) excluded by default (--include-pathological to run)" + ); + } + Console.WriteLine(); + + foreach (FixtureResult result in report.Results) + { + string status = result.Outcome switch + { + FixtureOutcome.Passed => "PASS", + FixtureOutcome.Failed => "FAIL", + FixtureOutcome.Skipped => "SKIP", + _ => "?", + }; + Console.WriteLine($"[{status}] {result.FixtureId} ({result.ElapsedMs}ms) {result.Reason}"); + if (result.Outcome == FixtureOutcome.Failed) + { + foreach (WordResult w in result.WordResults.Where(w => !w.Passed)) + Console.WriteLine($" word '{w.Word}': {w.Detail}"); + } + } + + Console.WriteLine(); + Console.WriteLine( + $"totals: {report.Passed} runs passed, {report.Failed} failed, {report.Skipped} skipped " + + $"(of {report.Results.Count} runs attempted across {report.FixtureCount} fixtures)" + ); + } + + /// Verifies the suite's checked-in fixture manifest byte for byte against a fresh computation, after + /// validating every grammar against the DTD and every words.yaml against its schema. The suite owns the + /// manifest and writes it with its own generator, so this never writes. + private static int RunCheckManifest(string suiteRoot) + { + string relative = ConformanceManifestGenerator.ManifestRelativePath; + string path = Path.Combine(suiteRoot, relative.Replace('/', Path.DirectorySeparatorChar)); + string dtd = Path.Combine(suiteRoot, ConformanceManifestGenerator.DtdRelativePath); + string wordsSchema = Path.Combine( + suiteRoot, + WordsSchemaValidation.SchemaRelativePath.Replace('/', Path.DirectorySeparatorChar) + ); + + ConformanceManifest corpus; + try + { + corpus = ConformanceManifestGenerator.Generate(suiteRoot); + } + catch (Exception ex) + when (ex is IOException or UnauthorizedAccessException or InvalidDataException or WordsYamlException) + { + Console.Error.WriteLine($"cannot compute the manifest under '{suiteRoot}': {ex.Message}"); + return 2; + } + + // Validate every fixture before comparing, so a rejected one is named rather than reported as stale. + foreach (ManifestFixture fixture in corpus.Fixtures) + { + IReadOnlyList messages; + try + { + messages = GrammarValidation.Validate( + Path.Combine(suiteRoot, fixture.GrammarPath.Replace('/', Path.DirectorySeparatorChar)), + dtd + ); + } + catch (Exception ex) when (ex is XmlException or IOException or UnauthorizedAccessException) + { + Console.Error.WriteLine($"{fixture.FixtureId}: grammar cannot be read: {ex.Message}"); + return 2; + } + if (messages.Count != 0) + { + Console.Error.WriteLine($"{fixture.FixtureId}: grammar does not validate against {corpus.DtdPath}"); + foreach (string message in messages) + Console.Error.WriteLine($" {message}"); + return 2; + } + + messages = WordsSchemaValidation.Validate( + Path.Combine(suiteRoot, fixture.WordsPath.Replace('/', Path.DirectorySeparatorChar)), + wordsSchema + ); + if (messages.Count == 0) + continue; + Console.Error.WriteLine( + $"{fixture.FixtureId}: words.yaml does not conform to {WordsSchemaValidation.SchemaRelativePath}" + ); + foreach (string message in messages) + Console.Error.WriteLine($" {message}"); + return 2; + } + + string json = ManifestJson.Serialize(corpus) + "\n"; + if (!File.Exists(path)) + { + Console.Error.WriteLine($"{path} is missing; regenerate it in the suite with scripts/generate_manifest.py"); + return 1; + } + + if (!string.Equals(File.ReadAllText(path).ReplaceLineEndings("\n"), json, StringComparison.Ordinal)) + { + Console.Error.WriteLine($"{path} is stale; regenerate it in the suite with scripts/generate_manifest.py"); + return 1; + } + + Console.WriteLine($"{path} is current ({corpus.Fixtures.Count} fixtures)"); + return 0; + } + + private const string OutOfScopeConstruct = "Tracing (TraceType)"; + + /// Recomputes coverage.csv and rules.csv without touching the fixtures root, then returns 1 if the + /// suite's checked-in copies are stale, an in-scope construct has no coverage, a blocked_by label is not a + /// grammar rule id, a rule no word exercises is missing from dead-rules-allowed.tsv, or an allowed rule + /// is no longer dead; 0 otherwise. The suite owns both CSVs, so they are written only into + /// , never into the fixtures root. + private static int RunCoverageReport( + List fixtures, + string fixturesRoot, + string constructsPath, + string coverageOutput + ) + { + bool failed = false; + string scratch = Path.Combine(Path.GetTempPath(), $"hc-coverage-report-{Guid.NewGuid():N}"); + Directory.CreateDirectory(scratch); + CoverageReport.CoverageResult result; + try + { + string freshCoverage = Path.Combine(scratch, "coverage.csv"); + string freshRules = Path.Combine(scratch, "rules.csv"); + result = CoverageReport.WriteCsvs(fixtures, freshCoverage, freshRules); + + Console.WriteLine(); + Console.WriteLine("coverage report"); + Console.WriteLine("==============="); + foreach (string name in new[] { "coverage.csv", "rules.csv" }) + { + string checkedIn = Path.Combine(fixturesRoot, name); + bool current = + File.Exists(checkedIn) + && string.Equals( + File.ReadAllText(checkedIn).ReplaceLineEndings("\n"), + File.ReadAllText(Path.Combine(scratch, name)).ReplaceLineEndings("\n"), + StringComparison.Ordinal + ); + Console.WriteLine($"{checkedIn} is {(current ? "current" : "STALE or missing")}"); + failed |= !current; + } + + if (coverageOutput != null) + { + string fixturesFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(fixturesRoot)); + string outputFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(coverageOutput)); + if ( + outputFull.Equals(fixturesFull, StringComparison.Ordinal) + || outputFull.StartsWith(fixturesFull + Path.DirectorySeparatorChar, StringComparison.Ordinal) + ) + { + Console.Error.WriteLine( + $"--coverage-output {coverageOutput} is inside the fixtures root; write into a separate clone of the suite, not the suite clone" + ); + return 2; + } + + Directory.CreateDirectory(outputFull); + foreach (string name in new[] { "coverage.csv", "rules.csv" }) + { + File.Copy(Path.Combine(scratch, name), Path.Combine(outputFull, name), overwrite: true); + Console.WriteLine($"wrote {Path.Combine(outputFull, name)}"); + } + } + } + finally + { + Directory.Delete(scratch, recursive: true); + } + + // Absolute construct-coverage check against constructs.txt: every construct except Tracing + // must be covered. + if (File.Exists(constructsPath)) + { + List checklist = CoverageReport.LoadConstructChecklist(constructsPath); + List uncovered = checklist + .Where(c => !string.Equals(c, OutOfScopeConstruct, StringComparison.Ordinal)) + .Where(c => !result.CoveredConstructs.Contains(c)) + .ToList(); + Console.WriteLine(); + int inScope = checklist.Count(c => !string.Equals(c, OutOfScopeConstruct, StringComparison.Ordinal)); + Console.WriteLine( + $"construct coverage: {inScope - uncovered.Count}/{inScope} in-scope constructs covered " + + $"(Tracing out of scope by design)" + ); + if (uncovered.Count > 0) + { + failed = true; + Console.WriteLine($"*** {uncovered.Count} CONSTRUCT(S) AT ZERO COVERAGE ***"); + foreach (string c in uncovered) + Console.WriteLine($" {c}"); + } + } + else + { + failed = true; + Console.WriteLine(); + Console.WriteLine($"*** CONSTRUCT CHECKLIST NOT FOUND: {constructsPath} ***"); + } + + if (result.UnknownBlockedBy.Count > 0) + { + failed = true; + Console.WriteLine(); + Console.WriteLine( + $"*** {result.UnknownBlockedBy.Count} BLOCKED_BY LABEL(S) ARE NOT RULE IDS IN THEIR FIXTURE'S GRAMMAR ***" + ); + foreach (CoverageReport.UnknownBlockedBy unknown in result.UnknownBlockedBy) + Console.WriteLine($" {unknown.FixtureId}: word '{unknown.Word}' blocked_by '{unknown.Label}'"); + } + + string allowlistPath = Path.Combine(fixturesRoot, "dead-rules-allowed.tsv"); + Dictionary<(string FixtureId, string RuleId), string> allowed = File.Exists(allowlistPath) + ? CoverageReport.LoadDeadRuleAllowlist(allowlistPath) + : new Dictionary<(string FixtureId, string RuleId), string>(); + (List unexpected, List<(string FixtureId, string RuleId)> stale) = + CoverageReport.CheckDeadRules(result.DeadRules, allowed); + + Console.WriteLine(); + Console.WriteLine( + $"dead rules: {result.DeadRules.Count} (exercised by zero words), " + + $"{result.DeadRules.Count - unexpected.Count} allowed by {Path.GetFileName(allowlistPath)}" + ); + if (unexpected.Count > 0) + { + failed = true; + Console.WriteLine($"*** {unexpected.Count} DEAD RULE(S) NOT IN {Path.GetFileName(allowlistPath)} ***"); + foreach (CoverageReport.DeadRule dead in unexpected) + Console.WriteLine($" {dead.FixtureId}: rule '{dead.RuleId}'"); + } + if (stale.Count > 0) + { + failed = true; + Console.WriteLine($"*** {stale.Count} ALLOWED RULE(S) NOW EXERCISED; REMOVE THEIR ENTRIES ***"); + foreach ((string fixtureId, string ruleId) in stale) + Console.WriteLine($" {fixtureId}: rule '{ruleId}'"); + } + return failed ? 1 : 0; + } + + private static void PrintUsage() + { + Console.WriteLine( + """ + usage: hc-conformance --fixtures [options] + + Runs every fixture under (the conformance/ clone, or conformance/stress) and compares + each word with its words.yaml: against the C# engine in process, or through --adapter. + + options: + --adapter "" Run fixtures through an external engine adapter. + Template placeholders: {grammar} {words} {output} + and {morpher}. Omit to run the harness's self-check mode + (in-process C# engine) instead. + --capabilities Capability set the engine under test declares (e.g. + "phonology", or "" for none). Defaults to the full set + in self-check mode, empty in --adapter mode. A fixture that + requires more is skipped. + --include-pathological Also run fixtures with a budget_ms (excluded by default). + --coverage-report Print a coverage report instead of running fixtures; + exits 1 on a stale coverage.csv or rules.csv, an + uncovered construct or an unallowed dead rule. Writes + nothing under --fixtures. + --coverage-output With --coverage-report, also write the fresh + coverage.csv and rules.csv into , which must be + outside --fixtures (a clone of the suite). + --constructs Construct checklist file for --coverage-report + (default: /constructs.txt). + --check-manifest Check the suite's generated/hc-conformance-manifest.v2.json against + a fresh computation, and every grammar and words.yaml against the + suite's DTD and schema. Writes nothing. + --propose Self-check only: on a signature mismatch, print the words.yaml + patch that would reconcile it. Never writes any file. + """ + ); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d8e979b31 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("SIL.Machine.Morphology.HermitCrab.Tests")] diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs new file mode 100644 index 000000000..3ac446c21 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/ProposePatchWriter.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// --propose: on a self-check signature mismatch, prints the words.yaml patch that +/// would reconcile the fixture with what the oracle actually produced, for a human to accept by +/// hand. This class only ever writes to the +/// it is given -- it has no file-write call anywhere, by construction, so +/// there is no code path here that could accidentally start writing words.yaml back out. +/// +public static class ProposePatchWriter +{ + public static void WriteSignatureProposal( + TextWriter output, + Fixture fixture, + WordEntry word, + List actualEntries, + bool wasSkipped + ) + { + output.WriteLine(); + output.WriteLine($"# --propose: {fixture.Id}/words.yaml -- word '{word.Word}' signature mismatch"); + output.WriteLine("# Proposed replacement entry (rules:/gloss: left as-is; fill in by hand):"); + if (actualEntries.Count == 0) + { + output.WriteLine($" - word: {word.Word}"); + output.WriteLine($" note: {word.Note}"); + // Self-check rejects expect_fail on a word the oracle skipped, so propose what it accepts. + output.WriteLine(wasSkipped ? " expect_skip: true" : " expect_fail: true"); + } + else + { + output.WriteLine($" - word: {word.Word}"); + output.WriteLine($" note: {word.Note}"); + output.WriteLine(" parses:"); + foreach (string entry in actualEntries) + { + output.WriteLine($" - signature: \"{entry}\""); + output.WriteLine(" rules: [] # TODO: fill in from the trace"); + } + } + output.WriteLine(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md new file mode 100644 index 000000000..1f0e6e3ca --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/README.md @@ -0,0 +1,109 @@ +# HermitCrab conformance runner + +`hc-conformance` runs the grammars of [sillsdev/fieldworks-parser-conformance](https://github.com/sillsdev/fieldworks-parser-conformance) +against machine's HermitCrab engine, in process. The suite is a clone in `conformance/` at the root of this repository, +at the one commit named in `conformance.commit`. Commands run from the repository root. `` below means +`dotnet run --project src/SIL.Machine.Morphology.HermitCrab.Conformance -c Release --`. + +## What runs where + +| Check | Where it runs | +|---|---| +| ` --fixtures conformance --include-pathological`, `--fixtures conformance/stress --include-pathological`, `--check-manifest`, `--coverage-report`, and that `conformance.commit` is on the suite's `main` | `.github/workflows/conformance.yml` and `./local_check.sh` | +| every fixture, the manifest, the DTD and schemas, the FieldWorks-producible shape rules | `dotnet test tests/SIL.Machine.Morphology.HermitCrab.Tests` (`ConformanceFixtureGateTests`, `ConformanceManifestTests`, `FieldWorksProducibleShapeTests`) | +| the 16 stress fixtures under NUnit | `dotnet test ... --filter "TestCategory=Stress"`, which is `[Explicit]` and never runs by default | + +## Getting the suite + +Nothing to do: building this project (which `dotnet build` and `dotnet test` of the HermitCrab tests do) runs +`EnsureConformanceSuite.targets`, which makes `conformance/` a clone of the suite at the commit in `conformance.commit`. + +- Absent: it is cloned (`git clone --filter=blob:none`, so the suite's history is available) into a temporary directory + beside `conformance/` and renamed, so an interrupted or parallel build never leaves a half-populated suite. +- At the pinned commit: nothing is fetched and nothing is written, so an offline build works. +- At another commit, for example after `git pull` changed the pin: moved to the pinned commit. If `conformance/` has + uncommitted changes the build stops and names the file; it never touches them. +- At the pinned commit with local changes (edited files, extra fixtures): the build warns and names them, and goes on, + so a suite author can try a change. The runner prints the same warning, and also warns when `conformance/` is not at + the pin, which `dotnet run --no-build` would otherwise read silently. +- No git, no network, a connection that stalls (git gives up after 20 seconds without data, the build after three + minutes in all), or any other failure: the build warns and never fails. The tests then fail with the two commands to + run by hand, `git clone --filter=blob:none https://github.com/sillsdev/fieldworks-parser-conformance.git conformance` + and `git -C conformance checkout --detach `. +- `SIL.Machine`, the HermitCrab library and the HermitCrab tool do not reference this project, so building or packing + them never touches `conformance/`. + +To run against a copy of the suite you are changing, set `HC_CONFORMANCE_SUITE=`. The tests then use +that directory, and the build neither clones nor checks the pin. The runner takes the same path with `--fixtures`. This +is how a change to the suite is tried here before it is merged there. + +Never edit, commit from or write into `conformance/` from machine. A change to a grammar, a `words.yaml`, the manifest, +`coverage.csv` or `rules.csv` is a pull request to the suite repository (see its `AGENTS.md`). No runner flag writes into +the suite. + +## Bumping the pin + +Do this when the suite has a change machine should run, and for no other reason. + +1. The change is merged on the suite's `main` and its CI is green. A pin must be on `main`: CI fails a pin that is not an + ancestor of the suite's `origin/main`. +2. Put the full 40-character commit SHA, nothing else, on the one line of `conformance.commit`. +3. Build once (`dotnet build tests/SIL.Machine.Morphology.HermitCrab.Tests`) so `conformance/` moves to it, then run the + checks under "Before you push". +4. Commit `conformance.commit` with a message that says what is now true, for example "Run the conformance suite at + ``, which adds the `` fixture". +5. If `--check-manifest` or `--coverage-report` reports one of the suite's own files as stale, that is a defect in the + suite. Fix it there and bump to that commit. Do not patch it in `conformance/`. + +## Regenerating what the suite owns + +The suite checks in files that are computed from its fixtures. The runner only checks them. + +- The manifest: in a clone of the suite, `python3 scripts/generate_manifest.py`. Machine's `--check-manifest` must agree + with it byte for byte. +- `coverage.csv` and `rules.csv`: ` --fixtures conformance --coverage-report --coverage-output ` + writes fresh copies into the clone. The output directory may not be inside `--fixtures`. +- The expected rows for a new or changed fixture: ` --fixtures --include-pathological --propose` prints + the `words.yaml` patch for each mismatch and writes nothing. Read every row against the rules before accepting it. The + `batch` command of `src/SIL.Machine.Morphology.HermitCrab.Tool` records the same signatures for a word list. + +## Running another engine + +`--adapter ""` runs the same fixtures through an external engine instead of the C# engine. The +template placeholders are `{grammar}`, `{words}`, `{output}` and `{morpher}`; the engine writes one row per word to +`{output}` in the format of `batch` in `src/SIL.Machine.Morphology.HermitCrab.Tool`, as the suite's `PROTOCOL.md` +describes. Pass `--capabilities phonology` (the default in adapter mode is none) so fixtures the engine can run are not +skipped. The template is split at spaces into the program and its arguments, the same way as a line of an +`hc --script-file`; a quote only groups words and is not passed on, so write `--grammar {grammar}`, not +`--grammar="{grammar}"`. Each placeholder is filled in after the split, so a path with spaces still reaches the engine +as one argument. For example, with a built PanGloss: + +``` + --fixtures conformance --capabilities phonology --adapter "/path/to/pangloss batch {grammar} {words} {output}" +``` + +The exit code is 0 when every run passes, 1 on a failure and 2 on a harness error; each failure is a `[FAIL] ` line. + +## Before you push + +Run each command from the repository root and read the output. Do not push on a partial run. + +``` +cat conformance.commit +git -C conformance rev-parse HEAD +pwsh ./scripts/comment-hygiene.ps1 +dotnet csharpier check . +dotnet build tests/SIL.Machine.Morphology.HermitCrab.Tests -c Release + --fixtures conformance --include-pathological + --fixtures conformance/stress --include-pathological + --fixtures conformance --check-manifest + --fixtures conformance --coverage-report +dotnet test tests/SIL.Machine.Morphology.HermitCrab.Tests -c Release +``` + +The two commits on the first line must match. `./local_check.sh --agent-strict` runs the repository-wide version of the +same checks, including that the pin is on the suite's `main`. + +In `conformance/`, `edge-cases/deep-optional-affix-nesting` carries a `budget_ms` of 15 s, and each stress fixture carries +60 to 120 s. Run Release builds; Debug is markedly slower. A loaded machine can still exceed one; rerun on a quiet +machine before concluding the engine regressed, and do not raise a budget here. Budgets are the suite's. diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs new file mode 100644 index 000000000..e999369be --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/RequiresDerivation.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.Linq; +using System.Xml; +using System.Xml.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public static class RequiresDerivation +{ + /// + /// phonology when the grammar has a phonological rule, plus + /// when any fixture run declares a morpher: block, in that order. + /// + public static List Derive(string grammarPath, MorpherSettings morpher) + { + return Derive(grammarPath, hasMorpherSettings: !morpher.IsEmpty); + } + + public static List Derive(string grammarPath, IEnumerable runs) + { + return Derive(grammarPath, hasMorpherSettings: runs.Any(settings => !settings.IsEmpty)); + } + + /// + /// Derive grammar capabilities and add the fixture-level settings capability when any run has + /// settings. A selected run's settings may be empty while another named run requires support. + /// + public static List Derive(string grammarPath, bool hasMorpherSettings) + { + List requires = Derive(grammarPath); + if (hasMorpherSettings) + requires.Add(MorpherSettings.Capability); + return requires; + } + + private static List Derive(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + bool hasPhonology = doc.Descendants() + .Any(e => e.Name.LocalName == "PhonologicalRule" || e.Name.LocalName == "MetathesisRule"); + + return hasPhonology ? new List { "phonology" } : new List(); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs new file mode 100644 index 000000000..47a131740 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/Runner.cs @@ -0,0 +1,449 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public class RunReport +{ + public List Results { get; } = new(); + public int FixtureCount { get; set; } + public int ExcludedPathologicalCount { get; set; } + + public int Passed => Results.Count(r => r.Outcome == FixtureOutcome.Passed); + public int Failed => Results.Count(r => r.Outcome == FixtureOutcome.Failed); + public int Skipped => Results.Count(r => r.Outcome == FixtureOutcome.Skipped); +} + +/// +/// Runs fixtures under conformance/languages and conformance/edge-cases. Self-check mode is a +/// custom, in-process loop (it needs per-word signature-SET comparison against words.yaml and +/// traced-rules verification, neither of which does). Adapter mode instead +/// materializes each fixture onto disk and hands it to +/// + -- see . +/// +public static class Runner +{ + /// The capabilities the C# engine declares. A fixture that requires another is skipped. + public static readonly IReadOnlySet DefaultCapabilities = new HashSet + { + "phonology", + MorpherSettings.Capability, + }; + + public static RunReport RunSelfCheck( + List fixtures, + bool includePathological, + IReadOnlySet engineCapabilities, + bool propose, + TextWriter proposeOutput + ) + { + var report = new RunReport(); + report.FixtureCount = fixtures.Count; + foreach (Fixture fixture in fixtures) + { + foreach (WordRun run in fixture.Words.Runs) + { + FixtureResult result = RunOneSelfCheck( + fixture, + run, + includePathological, + engineCapabilities, + propose, + proposeOutput + ); + if (result == null) + report.ExcludedPathologicalCount++; + else + report.Results.Add(result); + } + } + return report; + } + + private static FixtureResult RunOneSelfCheck( + Fixture fixture, + WordRun run, + bool includePathological, + IReadOnlySet engineCapabilities, + bool propose, + TextWriter proposeOutput + ) + { + string runId = run.Name == "default" ? fixture.Id : $"{fixture.Id} [variant {run.Name}]"; + var result = new FixtureResult { FixtureId = runId }; + + List derivedRequires = RequiresDerivation.Derive( + fixture.GrammarPath, + fixture.Words.Runs.Select(r => r.Morpher) + ); + if ( + !new HashSet(derivedRequires, StringComparer.Ordinal).SetEquals( + new HashSet(fixture.Words.Requires, StringComparer.Ordinal) + ) + ) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = + $"words.yaml error: requires [{string.Join(",", fixture.Words.Requires)}] does not match " + + $"grammar-derived [{string.Join(",", derivedRequires)}]"; + return result; + } + + bool isPathological = fixture.Words.BudgetMs.HasValue; + if (isPathological && !includePathological) + return null; + + List missingCapabilities = fixture.Words.Requires.Where(r => !engineCapabilities.Contains(r)).ToList(); + if (missingCapabilities.Count > 0) + { + result.Outcome = FixtureOutcome.Skipped; + result.Reason = + $"requires [{string.Join(",", fixture.Words.Requires)}], missing [{string.Join(",", missingCapabilities)}]"; + return result; + } + + var sw = Stopwatch.StartNew(); + try + { + RunWithBudget(fixture, run, result, propose, proposeOutput); + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + + if (fixture.Words.ExpectCrash) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = "expected crash, engine produced output"; + } + else + { + bool overBudget = fixture.Words.BudgetMs.HasValue && result.ElapsedMs > fixture.Words.BudgetMs.Value; + if (overBudget) + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"exceeded wall-clock budget ({result.ElapsedMs}ms > {fixture.Words.BudgetMs}ms)"; + } + else if (result.WordResults.All(w => w.Passed)) + { + result.Outcome = FixtureOutcome.Passed; + } + else + { + result.Outcome = FixtureOutcome.Failed; + // Name the words. A bare count cannot be diagnosed from a CI log on a machine you cannot reach. + WordResult[] mismatched = result.WordResults.Where(word => !word.Passed).ToArray(); + int total = result.WordResults.Count; + string named = string.Join( + "; ", + mismatched.Take(5).Select(word => $"{word.Word}: {Truncate(word.Detail)}") + ); + string more = mismatched.Length > 5 ? $"; and {mismatched.Length - 5} more" : ""; + result.Reason = $"{mismatched.Length}/{total} word(s) mismatched: {named}{more}"; + } + } + } + catch (Exception ex) + { + sw.Stop(); + result.ElapsedMs = sw.ElapsedMilliseconds; + // As in adapter mode, only the engine's own crash matches expect_crash; a timeout or a harness + // failure is a failure. + if (fixture.Words.ExpectCrash && ex is EngineCrashException) + { + result.Outcome = FixtureOutcome.Passed; + result.Reason = $"engine crashed as expected (words.yaml expect_crash: true): {ex.Message}"; + } + else + { + result.Outcome = FixtureOutcome.Failed; + result.Reason = $"engine crashed in run '{run.Name}': {ex.Message}"; + } + } + + return result; + } + + // Mirrors MaterializedRunner.RunWithBudget's watchdog shape (see that method's doc comment) -- a + // budgeted fixture gets the same "background thread + Task.Wait" timeout enforcement. + private static void RunWithBudget( + Fixture fixture, + WordRun run, + FixtureResult result, + bool propose, + TextWriter proposeOutput + ) + { + if (!fixture.Words.BudgetMs.HasValue) + { + RunAllWords(fixture, run, result.WordResults, propose, proposeOutput, CancellationToken.None); + return; + } + + long budgetMs = fixture.Words.BudgetMs.Value; + // The task fills its own list, so a task abandoned on timeout cannot mutate the reported result. + var wordResults = new List(); + using var cancellation = new CancellationTokenSource(); + Task task = Task.Run(() => RunAllWords(fixture, run, wordResults, propose, proposeOutput, cancellation.Token)); + long marginMs = Math.Max(budgetMs / 5, 1000); + bool completed; + try + { + completed = task.Wait((int)(budgetMs + marginMs)); + } + catch (AggregateException ex) when (ex.InnerException != null) + { + throw ex.InnerException; + } + if (!completed) + { + cancellation.Cancel(); + throw new TimeoutException("watchdog timed out waiting for self-check engine"); + } + result.WordResults.AddRange(wordResults); + } + + private static void RunAllWords( + Fixture fixture, + WordRun run, + List wordResults, + bool propose, + TextWriter proposeOutput, + CancellationToken cancellationToken + ) + { + // Only a failure of the engine itself is an EngineCrashException, so expect_crash cannot excuse a + // harness failure such as the rule index rejecting a grammar. + Morpher morpher; + try + { + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + morpher = ConformanceMorpherFactory.CreateTracing(language, run.Morpher); + } + catch (Exception ex) + { + throw new EngineCrashException($"engine failed to load '{fixture.GrammarPath}': {ex.Message}", ex); + } + GrammarRuleIndex ruleIndex = GrammarRuleIndex.Load(fixture.GrammarPath); + + foreach (WordEntry word in run.Words) + { + cancellationToken.ThrowIfCancellationRequested(); + bool guessRoot = run.Morpher.GuessRootsFor(word); + List results; + object trace = null; + bool wasSkipped = false; + try + { + results = morpher.ParseWord(word.Word, out trace, guessRoot).ToList(); + } + catch (InvalidShapeException) + { + wasSkipped = true; + results = new List(); + } + catch (Exception ex) + { + throw new EngineCrashException($"engine failed to parse '{word.Word}': {ex.Message}", ex); + } + + string actualSignature = SignatureFormat.BuildSignature(results); + string expectedSignature = FixtureMaterializer.BuildExpectedSignature(word); + List actualEntries = SignatureTsv.SplitSignature(actualSignature); + List expectedEntries = SignatureTsv.SplitSignature(expectedSignature); + + bool signatureMatches = SignatureTsv + .SplitComparableSignature(actualSignature) + .SequenceEqual(SignatureTsv.SplitComparableSignature(expectedSignature), StringComparer.Ordinal); + if (!signatureMatches) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = false, + Detail = + $"expected [{string.Join(" | ", expectedEntries)}] got [{string.Join(" | ", actualEntries)}]", + } + ); + if (propose && run.Name == "default") + { + ProposePatchWriter.WriteSignatureProposal(proposeOutput, fixture, word, actualEntries, wasSkipped); + } + else if (propose) + { + proposeOutput.WriteLine( + $"# --propose cannot safely patch variant '{run.Name}' in {fixture.Id}; edit that variant's complete words list by hand." + ); + } + continue; + } + + if (word.ExpectSkip) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = wasSkipped, + Detail = wasSkipped + ? "ok (skipped: InvalidShapeException)" + : "declared expect_skip but engine produced a zero-parse 'ok' without skipping", + } + ); + continue; + } + + if (word.ExpectFail) + { + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = !wasSkipped, + Detail = !wasSkipped + ? "ok (0 parses)" + : "declared expect_fail (status ok) but engine SKIPPED it (InvalidShapeException) -- use expect_skip", + } + ); + continue; + } + + HashSet wordLevelRuleIds = TraceRuleAttributor.WordLevelRuleIds(trace, ruleIndex); + + List ruleDetails = CompareRuleTraceMultisets( + word.Parses.Select(parse => (parse.Signature, (IEnumerable)parse.Rules)), + results.Select(match => + { + string signature = SignatureFormat.BuildSignature(new[] { match }); + HashSet actualRuleIds = TraceRuleAttributor.MorphologicalRuleIds(match, ruleIndex); + actualRuleIds.UnionWith(wordLevelRuleIds); + return (signature, (IEnumerable)actualRuleIds); + }) + ); + bool wordRulesOk = ruleDetails.Count == 0; + wordResults.Add( + new WordResult + { + Word = word.Word, + Passed = wordRulesOk, + Detail = wordRulesOk ? "ok" : string.Join("; ", ruleDetails), + } + ); + } + } + + internal static List CompareRuleTraceMultisets( + IEnumerable<(string Signature, IEnumerable RuleIds)> expected, + IEnumerable<(string Signature, IEnumerable RuleIds)> actual + ) + { + static string CanonicalRuleIds(IEnumerable ruleIds) => + string.Join( + '\u001f', + ruleIds.Distinct(StringComparer.Ordinal).OrderBy(ruleId => ruleId, StringComparer.Ordinal) + ); + + static Dictionary<(string Signature, string RuleIds), int> CountTraces( + IEnumerable<(string Signature, IEnumerable RuleIds)> traces + ) + { + var counts = new Dictionary<(string Signature, string RuleIds), int>(); + foreach ((string signature, IEnumerable ruleIds) in traces) + { + var key = (SignatureTsv.CanonicalizeBracketClasses(signature), CanonicalRuleIds(ruleIds)); + counts[key] = counts.GetValueOrDefault(key) + 1; + } + return counts; + } + + Dictionary<(string Signature, string RuleIds), int> expectedCounts = CountTraces(expected); + Dictionary<(string Signature, string RuleIds), int> actualCounts = CountTraces(actual); + var details = new List(); + foreach ( + (string Signature, string RuleIds) key in expectedCounts + .Keys.Union(actualCounts.Keys) + .OrderBy(key => key.Signature, StringComparer.Ordinal) + .ThenBy(key => key.RuleIds, StringComparer.Ordinal) + ) + { + int declaredCount = expectedCounts.GetValueOrDefault(key); + int tracedCount = actualCounts.GetValueOrDefault(key); + if (declaredCount == tracedCount) + continue; + + string declaredRuleIds = string.Join( + ',', + key.RuleIds.Length == 0 ? Array.Empty() : key.RuleIds.Split('\u001f') + ); + details.Add( + $"parse '{key.Signature}': rules [{declaredRuleIds}]: declared {declaredCount}, " + + $"traced {tracedCount}" + ); + } + + return details; + } + + // ---- Adapter mode: materialize onto disk + reuse MaterializedRunner/AdapterEngine, unmodified. ---- + + public static RunReport RunAdapter(List fixtures, IEngine engine, bool includePathological) + { + var report = new RunReport(); + report.FixtureCount = fixtures.Count; + // One call over every run of every fixture, so an expect_crash fixture has an ordinary fixture to use + // as its control. + var materialized = new List(); + try + { + foreach (Fixture fixture in fixtures) + { + foreach (WordRun run in fixture.Words.Runs) + { + try + { + materialized.Add(FixtureMaterializer.Materialize(fixture, run)); + } + catch (NamedVariantHasNoAdapterVisibleWordsException ex) + { + report.Results.Add( + new FixtureResult + { + FixtureId = run.Name == "default" ? fixture.Id : $"{fixture.Id} [variant {run.Name}]", + Outcome = FixtureOutcome.Failed, + Reason = ex.Message, + } + ); + } + } + } + + MaterializedRunReport runReport = MaterializedRunner.Run(materialized, engine, includePathological); + report.ExcludedPathologicalCount = runReport.ExcludedPathologicalCount; + report.Results.AddRange(runReport.Results); + } + finally + { + foreach (MaterializedFixture fixture in materialized) + FixtureMaterializer.Cleanup(fixture); + } + return report; + } + + /// Shortens a per-word detail so a fixture reason stays readable in a log. + /// The word detail, which may carry every expected and actual parse. + private static string Truncate(string detail) + { + const int Limit = 160; + if (string.IsNullOrEmpty(detail) || detail.Length <= Limit) + { + return detail; + } + + return detail[..Limit] + "..."; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj new file mode 100644 index 000000000..3d8818af0 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SIL.Machine.Morphology.HermitCrab.Conformance.csproj @@ -0,0 +1,32 @@ + + + + Exe + net10.0 + SIL.Machine.Morphology.HermitCrab.Conformance + hc-conformance + Runs the HermitCrab conformance suite (the clone in conformance/) against the C# engine. + SIL.Machine.Morphology.HermitCrab.Conformance + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + + + diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs new file mode 100644 index 000000000..3aaf3106d --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SignatureTsv.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +public readonly struct TsvRow(string word, string status, string signature) +{ + public readonly string Word = word; + public readonly string Status = status; + public readonly string Signature = signature; +} + +public static class SignatureTsv +{ + /// Reads a batch TSV file, silently skipping malformed/sentinel lines. + public static List ReadFile(string path) + { + var rows = new List(); + foreach (string line in File.ReadAllLines(path)) + { + if (TryParseRow(line, out TsvRow row)) + rows.Add(row); + } + return rows; + } + + public static bool TryParseRow(string line, out TsvRow row) + { + row = default; + if (string.IsNullOrEmpty(line)) + return false; + string[] cols = line.Split('\t'); + // A well-formed row has exactly 5 columns (idx, word, ms, status, signature). The + // 3-column "STARTED" sentinel and any other malformed line is not comparison data. + if (cols.Length != 5) + return false; + if (!int.TryParse(cols[0], out _)) + return false; + if (!long.TryParse(cols[2], out _)) + return false; + + if (cols[3] != "ok" && cols[3] != "SKIPPED") + return false; + row = new TsvRow(cols[1], cols[3], cols[4]); + return true; + } + + /// The ";"-separated signature entries as a multiset (sorted list, for equality/diff display). + public static List SplitSignature(string signature) + { + if (signature == "-") + return new List(); + return signature.Split(';').OrderBy(s => s, StringComparer.Ordinal).ToList(); + } + + private static readonly Regex BracketClass = new(@"\[(\^?)([^\[\]]*)\]", RegexOptions.Compiled); + + /// + /// with the members of every bracketed class sorted, so "[ea]" equals "[ae]". + /// HermitCrab renders class members in character-table insertion order, which FieldWorks fills from an + /// unordered collection (sillsdev/machine#573). + /// + public static List SplitComparableSignature(string signature) => + SplitSignature(signature).Select(CanonicalizeBracketClasses).OrderBy(s => s, StringComparer.Ordinal).ToList(); + + /// One signature entry with the members of every bracketed class sorted. + public static string CanonicalizeBracketClasses(string entry) => + BracketClass.Replace( + entry, + m => "[" + m.Groups[1].Value + new string(m.Groups[2].Value.OrderBy(c => c).ToArray()) + "]" + ); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs new file mode 100644 index 000000000..d4982f6ff --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/SuitePinCheck.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Warns when a run reads the repository's conformance/ clone while it is not at the commit in +/// conformance.commit, which dotnet run --no-build would otherwise do silently after a pin edit, +/// or while it holds local changes. A --fixtures path outside such a clone, or +/// HC_CONFORMANCE_SUITE being set, means the caller chose the suite and nothing is checked. +/// +internal static class SuitePinCheck +{ + private const string OverrideVariable = "HC_CONFORMANCE_SUITE"; + + public static IReadOnlyList Warnings(string fixturesRoot) + { + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(OverrideVariable))) + return []; + + string pinFile = null; + string suite = null; + for ( + DirectoryInfo directory = new DirectoryInfo(Path.GetFullPath(fixturesRoot)); + directory?.Parent != null; + directory = directory.Parent + ) + { + string candidate = Path.Combine(directory.Parent.FullName, "conformance.commit"); + if (directory.Name == "conformance" && File.Exists(candidate)) + { + pinFile = candidate; + suite = directory.FullName; + break; + } + } + if (suite == null || !Directory.Exists(Path.Combine(suite, ".git"))) + return []; + + string pin = File.ReadAllText(pinFile).Trim(); + string head = Git(suite, "rev-parse", "HEAD")?.Trim(); + if (head == null) + return [$"could not run git in {suite}, so it was not checked against conformance.commit ({pin})"]; + + var warnings = new List(); + if (head != pin) + { + warnings.Add( + $"{suite} is at {head}, not the commit in conformance.commit ({pin}), so this run reads a stale suite. " + + "Build the runner (dotnet build src/SIL.Machine.Morphology.HermitCrab.Conformance) to move it." + ); + } + + string status = Git(suite, "status", "--porcelain"); + string[] changed = (status ?? "") + .Split('\n') + .Select(line => line.TrimEnd('\r')) + .Where(line => line.Length > 3) + .Select(line => line.Substring(3)) + .ToArray(); + if (changed.Length > 0) + { + warnings.Add( + $"{suite} has {changed.Length} local change(s): {string.Join(", ", changed.Take(5))}" + + (changed.Length > 5 ? ", ..." : "") + + ". This run uses what is on disk." + ); + } + return warnings; + } + + private static string Git(string workingDirectory, params string[] arguments) + { + var startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = workingDirectory; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.EnvironmentVariables["GIT_OPTIONAL_LOCKS"] = "0"; + foreach (string variable in new[] { "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE" }) + startInfo.EnvironmentVariables.Remove(variable); + foreach (string argument in arguments) + startInfo.ArgumentList.Add(argument); + try + { + using var process = System.Diagnostics.Process.Start(startInfo); + string output = process.StandardOutput.ReadToEnd(); + process.StandardError.ReadToEnd(); + process.WaitForExit(); + return process.ExitCode == 0 ? output : null; + } + catch (System.ComponentModel.Win32Exception) + { + return null; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs new file mode 100644 index 000000000..e05a250d6 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/TraceRuleAttributor.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Extracts, for one successful parse () returned by Morpher.ParseWord, +/// the set of grammar rule ids a word's "rules:" field is verified against: self-check runs the +/// oracle with tracing enabled and fails if the traced rule applications don't match the declared +/// list. +/// +/// Two different sources, because no single API gives both: +/// +/// Morphological rules (AffixProcessRule/CompoundingRule) come straight off +/// -- populated once during analysis-direction unapplication +/// and carried forward unchanged through synthesis, so it is exact and per-result, no ambiguity. +/// Phonological rules (RewriteRule/MetathesisRule) have no equivalent per-Word list -- +/// only records their application as nodes +/// (), via . +/// Unlike , that call does NOT update +/// Word.CurrentTrace (see TraceManager.cs), so a phonological-rule trace node is a *sibling* +/// of the branch that produced a given final Word, not an ancestor reachable by walking +/// CurrentTrace upward. This class instead walks the ENTIRE trace tree for the word (the +/// trace out-parameter of ParseWord) and collects every +/// node system-wide. +/// +/// +/// Documented limitation (ambiguous words): when a surface word has more than one successful +/// analysis, this global trace walk cannot attribute a specific phonological rule firing to one +/// analysis branch over another -- it reports the UNION of every phonological rule that fired +/// anywhere while producing any of the word's results. applies this same +/// phonological-rule set to every one of that word's declared parses, so a false PASS is possible +/// in principle for an ambiguous word whose two parses use different phonological rules but the +/// same declared "rules:" set does not distinguish them. This does not affect any G1 fixture (the +/// pilot grammar has zero phonological rules by construction; the one ambiguous pilot word's +/// parses differ only in a morphological rule, which IS attributed exactly per-result). A future +/// language with both ambiguity and phonological rules would need per-branch trace correlation +/// (e.g. threading a distinguishing marker through ITraceManager.PhonologicalRuleApplied) to +/// close this gap -- flagged here rather than silently shipped. +/// +/// RealizationalAffixProcessRule applications are attributed word-level too: +/// Word.MorphologicalRuleUnapplied deliberately excludes them from the _mruleApps +/// list that backs (see Word.cs), so they cannot come from +/// the per-result source above -- but SynthesisRealizationalAffixProcessRule does report +/// successful applications through , which +/// records a node. The same trace walk that +/// collects phonological rules therefore also collects realizational rules (with the same +/// union-across-results ambiguity caveat), keyed off the node's Source type. +/// +public static class TraceRuleAttributor +{ + public static HashSet MorphologicalRuleIds(Word result, GrammarRuleIndex index) + { + var ids = new HashSet(StringComparer.Ordinal); + foreach (IMorphologicalRule rule in result.MorphologicalRules) + { + string id = index.ResolveMorphologicalRuleId(rule); + if (id != null) + ids.Add(id); + } + return ids; + } + + /// The word-level (not per-result) set of phonological and realizational rule ids fired + /// anywhere in the trace tree rooted at -- see the class doc comment for + /// why these two kinds are word-level, not per-result. + public static HashSet WordLevelRuleIds(object trace, GrammarRuleIndex index) + { + var ids = new HashSet(StringComparer.Ordinal); + if (trace is not Trace root) + return ids; + Walk(root); + return ids; + + void Walk(Trace node) + { + if (node.FailureReason == FailureReason.None) + { + if (node.Type == TraceType.PhonologicalRuleSynthesis && node.Source is IPhonologicalRule prule) + { + string id = index.ResolvePhonologicalRuleId(prule); + if (id != null) + ids.Add(id); + } + else if ( + node.Type == TraceType.MorphologicalRuleSynthesis + && node.Source is RealizationalAffixProcessRule rrule + ) + { + string id = index.ResolveMorphologicalRuleId(rrule); + if (id != null) + ids.Add(id); + } + } + foreach (Trace child in node.Children) + Walk(child); + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs new file mode 100644 index 000000000..ac93197ce --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsSchemaValidation.cs @@ -0,0 +1,142 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json.Nodes; +using Json.Schema; +using YamlDotNet.RepresentationModel; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// Validates an authored words.yaml against the published words schema. +/// The schema, not the loader, is what a consumer outside this repository can read, so the +/// two must agree on every checked-in fixture. Plain YAML only: aliases, anchors, merge keys and +/// custom tags are refused before conversion rather than silently resolved. +public static class WordsSchemaValidation +{ + public const string SchemaRelativePath = "schema/words.schema.json"; + + // A schema registers itself globally under its $id, so it is loaded once per path rather than + // once per file validated. + private static readonly Dictionary LoadedSchemas = new(StringComparer.OrdinalIgnoreCase); + + /// Validation messages for ; empty when it conforms. + public static IReadOnlyList Validate(string wordsPath, string schemaPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(wordsPath); + ArgumentException.ThrowIfNullOrWhiteSpace(schemaPath); + + JsonNode? node = ReadAsJson(wordsPath); + if (node is null) + return new[] { "words.yaml is empty" }; + + using var document = System.Text.Json.JsonDocument.Parse(node.ToJsonString()); + EvaluationResults results = Schema(schemaPath) + .Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); + if (results.IsValid) + return Array.Empty(); + + return (results.Details ?? new List()) + .Where(detail => detail.Errors is { Count: > 0 }) + .SelectMany(detail => detail.Errors!.Select(error => $"{detail.InstanceLocation}: {error.Value}")) + .DefaultIfEmpty("does not conform to " + SchemaRelativePath) + .ToArray(); + } + + private static JsonSchema Schema(string schemaPath) + { + string key = Path.GetFullPath(schemaPath); + lock (LoadedSchemas) + { + if (!LoadedSchemas.TryGetValue(key, out JsonSchema? schema)) + { + schema = JsonSchema.FromFile(key); + LoadedSchemas[key] = schema; + } + + return schema; + } + } + + private static JsonNode? ReadAsJson(string wordsPath) + { + var stream = new YamlStream(); + using (var reader = new StreamReader(wordsPath)) + stream.Load(reader); + if (stream.Documents.Count == 0) + return null; + if (stream.Documents.Count != 1) + throw new InvalidDataException($"{wordsPath}: a words file must contain exactly one YAML document."); + + return Convert(stream.Documents[0].RootNode, wordsPath); + } + + private static JsonNode? Convert(YamlNode node, string path) + { + if (!node.Anchor.IsEmpty) + throw new InvalidDataException($"{path}: YAML anchors are not admitted."); + + switch (node) + { + case YamlMappingNode mapping: + { + var result = new JsonObject(); + foreach (KeyValuePair pair in mapping.Children) + { + if (pair.Key is not YamlScalarNode { Value: { } key }) + throw new InvalidDataException($"{path}: mapping keys must be scalars."); + if (key == "<<") + throw new InvalidDataException($"{path}: YAML merge keys are not admitted."); + result[key] = Convert(pair.Value, path); + } + + return result; + } + + case YamlSequenceNode sequence: + { + var result = new JsonArray(); + foreach (YamlNode child in sequence.Children) + result.Add(Convert(child, path)); + return result; + } + + case YamlScalarNode scalar: + return ConvertScalar(scalar); + + default: + throw new InvalidDataException($"{path}: unsupported YAML node '{node.GetType().Name}'."); + } + } + + /// Applies YAML 1.1 core scalar resolution for the types the schema distinguishes. + private static JsonNode? ConvertScalar(YamlScalarNode scalar) + { + string? value = scalar.Value; + if (value is null) + return null; + if ( + scalar.Style + is YamlDotNet.Core.ScalarStyle.SingleQuoted + or YamlDotNet.Core.ScalarStyle.DoubleQuoted + or YamlDotNet.Core.ScalarStyle.Literal + or YamlDotNet.Core.ScalarStyle.Folded + ) + { + return JsonValue.Create(value); + } + + if (value is "null" or "~" or "") + return null; + if (value is "true" or "True" or "TRUE") + return JsonValue.Create(true); + if (value is "false" or "False" or "FALSE") + return JsonValue.Create(false); + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out long integer)) + return JsonValue.Create(integer); + + return JsonValue.Create(value); + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs new file mode 100644 index 000000000..eb95d20a7 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYaml.cs @@ -0,0 +1,94 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// In-memory model of one words.yaml file. Parsed by , which +/// enforces the strict, fixed key vocabulary -- this class only carries already-validated data, it +/// does not itself validate anything. +/// +public class WordsYaml +{ + public string Language = ""; + public List InspiredBy { get; } = new(); + public List Sources { get; } = new(); + public List Requires { get; } = new(); + + /// Edge-cases only: a wall-clock budget in milliseconds for the whole fixture. + public long? BudgetMs; + + /// Edge-cases only: this fixture's ground truth is a crash, not a signature. + public bool ExpectCrash; + + public bool? FieldworksProducible; + + /// Required and non-empty when is false: names the + /// offending construct(s) and cites the HCLoader evidence grounding the verdict. + public string FieldworksProducibleNotes = ""; + + /// The morpher: block; empty when absent, meaning the engine constructor defaults. + public MorpherSettings Morpher { get; } = new(); + + public List Words { get; } = new(); + + /// + /// Named complete runs against the same grammar; coverage remains attached to . + /// + public List Variants { get; } = new(); + + public IEnumerable Runs + { + get + { + yield return new WordRun("default", Morpher, Words); + foreach (WordRun variant in Variants) + yield return variant; + } + } +} + +public class WordRun(string name, MorpherSettings morpher, List words) +{ + public string Name { get; } = name; + public MorpherSettings Morpher { get; } = morpher; + public List Words { get; } = words; +} + +public class WordEntry +{ + public string Word = ""; + public string Note = ""; + public string Provenance = ""; + public bool ExpectFail; + + /// + /// The oracle SKIPS this word rather than returning a genuine zero-parse "ok": it throws + /// (e.g. the surface contains a segment undeclared in the + /// grammar's CharacterDefinitionTable). Distinct from (well-formed input, + /// zero valid analyses) because the two produce different status columns -- + /// "SKIPPED" vs "ok" -- in BatchCommand's output. The runner confirms an + /// was actually thrown. Like expect_fail, a skip word has no parses. + /// + public bool ExpectSkip; + + public List BlockedBy { get; } = new(); + + /// Declaration ids (a grammar element's id) whose deactivation this word's failure observes. + /// The runner does not read it; the loader accepts it because the suite's schema allows it. + public List Neutralizes { get; } = new(); + public List Exercises { get; } = new(); + public List Parses { get; } = new(); + + /// The cell of each claims entry. The loader checks the entry's shape, since the suite's + /// schema allows it; the runner does not otherwise read it. + public List Claims { get; } = new(); +} + +public class ParseEntry +{ + public string Signature = ""; + public string Gloss = ""; + public bool Guess; + public List Rules { get; } = new(); + public List Exercises { get; } = new(); +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs new file mode 100644 index 000000000..78a182ea5 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Conformance/WordsYamlLoader.cs @@ -0,0 +1,512 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.RepresentationModel; + +namespace SIL.Machine.Morphology.HermitCrab.Conformance; + +/// +/// Strict parser for a fixture's words.yaml: enforces a fixed key vocabulary (unknown keys +/// are hard errors) and a plain YAML 1.2 subset only -- no anchors, aliases, merge keys, or custom +/// tags. +/// +public static class WordsYamlLoader +{ + // The complete, fixed key vocabulary at each of the three mapping levels. Anything else + // encountered is a hard parse error -- this is the mechanism behind "unknown keys are errors". + private static readonly HashSet FrontMatterKeys = new(StringComparer.Ordinal) + { + "language", + "inspired_by", + "sources", + "requires", + "budget_ms", + "expect_crash", + "fieldworks_producible", + "fieldworks_producible_notes", + "morpher", + "words", + "variants", + }; + + // The morpher: block's keys are FieldWorks element names (HCParser.LoadParser), + // limited to the settings whose only effect is which analyses a parse returns. + private static readonly HashSet MorpherKeys = new(StringComparer.Ordinal) + { + "DelReapps", + "MaxRoots", + "MergeAnalyses", + "MakePartialsFinal", + "GuessRoots", + }; + private static readonly HashSet WordKeys = new(StringComparer.Ordinal) + { + "word", + "note", + "provenance", + "expect_fail", + "expect_skip", + "blocked_by", + "neutralizes", + "exercises", + "parses", + "claims", + }; + private static readonly HashSet ParseKeys = new(StringComparer.Ordinal) + { + "signature", + "gloss", + "guess", + "rules", + "exercises", + }; + private static readonly HashSet ClaimKeys = new(StringComparer.Ordinal) + { + "cell", + "severing", + "before", + "after", + "distinct_from", + "proof", + }; + + public static WordsYaml Load(string path) + { + string text = File.ReadAllText(path); + try + { + RejectNonPlainConstructs(text, path); + + var yamlStream = new YamlStream(); + using (var reader = new StringReader(text)) + yamlStream.Load(reader); + + if (yamlStream.Documents.Count != 1) + throw new WordsYamlException(path, "expected exactly one YAML document"); + + if (yamlStream.Documents[0].RootNode is not YamlMappingNode root) + throw new WordsYamlException(path, "top-level YAML node must be a mapping"); + + return ParseFrontMatter(root, path); + } + catch (YamlException ex) + { + throw new WordsYamlException(path, $"YAML syntax error at {ex.Start}-{ex.End}: {ex.Message}", ex); + } + } + + /// + /// Scans the raw parser event stream (not the composed node tree -- by the time + /// composes a tree, an alias has already been resolved into a shared + /// node reference and is no longer visible as such) for anchors, aliases, and non-core tags, + /// which words.yaml forbids along with merge keys and custom tags. Merge keys ("<<") + /// are a mapping-key convention layered on plain scalars, not a distinct event type, so those + /// are rejected in instead, where key strings are actually read. + /// + private static void RejectNonPlainConstructs(string text, string path) + { + var parser = new Parser(new StringReader(text)); + while (parser.MoveNext()) + { + ParsingEvent evt = parser.Current; + if (evt is AnchorAlias) + throw new WordsYamlException(path, "YAML aliases (*name) are not allowed in words.yaml"); + + if (evt is NodeEvent nodeEvent) + { + if (!nodeEvent.Anchor.IsEmpty) + throw new WordsYamlException(path, "YAML anchors (&name) are not allowed in words.yaml"); + if ( + !nodeEvent.Tag.IsEmpty + && !nodeEvent.Tag.Value.StartsWith("tag:yaml.org,2002:", StringComparison.Ordinal) + ) + { + throw new WordsYamlException( + path, + $"custom YAML tag '{nodeEvent.Tag.Value}' is not allowed in words.yaml" + ); + } + } + } + } + + private static WordsYaml ParseFrontMatter(YamlMappingNode root, string path) + { + var doc = new WordsYaml(); + Dictionary map = RequireMapping(root, FrontMatterKeys, path, "front matter"); + + doc.Language = RequireScalar(map, "language", path, "front matter"); + AppendScalarList(map, "inspired_by", doc.InspiredBy, path, "front matter"); + AppendScalarList(map, "sources", doc.Sources, path, "front matter"); + AppendScalarList(map, "requires", doc.Requires, path, "front matter"); + if (map.TryGetValue("budget_ms", out YamlNode budgetNode)) + { + string budget = RequireScalarNode(budgetNode, path, "front matter.budget_ms"); + if (!long.TryParse(budget, NumberStyles.None, CultureInfo.InvariantCulture, out long budgetMs)) + { + throw new WordsYamlException( + path, + $"front matter.budget_ms: expected a whole number of milliseconds, got '{budget}'" + ); + } + doc.BudgetMs = budgetMs; + } + if (map.TryGetValue("expect_crash", out YamlNode crashNode)) + doc.ExpectCrash = ParseBool(RequireScalarNode(crashNode, path, "front matter.expect_crash"), path); + if (map.TryGetValue("fieldworks_producible", out YamlNode fwProducibleNode)) + { + doc.FieldworksProducible = ParseBool( + RequireScalarNode(fwProducibleNode, path, "front matter.fieldworks_producible"), + path + ); + } + if (map.TryGetValue("fieldworks_producible_notes", out YamlNode fwNotesNode)) + { + doc.FieldworksProducibleNotes = RequireScalarNode( + fwNotesNode, + path, + "front matter.fieldworks_producible_notes" + ); + } + if (doc.FieldworksProducible == false && doc.FieldworksProducibleNotes.Length == 0) + { + throw new WordsYamlException( + path, + "front matter: 'fieldworks_producible: false' requires a non-empty 'fieldworks_producible_notes'" + ); + } + + if (map.TryGetValue("morpher", out YamlNode morpherNode)) + ParseMorpher(morpherNode, doc.Morpher, path); + + if (!map.TryGetValue("words", out YamlNode wordsNode) || wordsNode is not YamlSequenceNode wordsSeq) + throw new WordsYamlException(path, "front matter must have a 'words' sequence"); + if (wordsSeq.Children.Count == 0) + throw new WordsYamlException(path, "'words' must contain at least one word entry"); + + foreach (YamlNode wordNode in wordsSeq.Children) + doc.Words.Add(ParseWord(wordNode, path)); + + if (map.TryGetValue("variants", out YamlNode variantsNode)) + ParseVariants(variantsNode, doc, path); + + bool requiresSettings = !doc.Morpher.IsEmpty || doc.Variants.Count > 0; + if (doc.Requires.Contains(MorpherSettings.Capability) != requiresSettings) + { + throw new WordsYamlException( + path, + $"front matter: requires must {(requiresSettings ? "contain" : "not contain")} '{MorpherSettings.Capability}' " + + "when a run declares morpher settings" + ); + } + + return doc; + } + + private static void ParseVariants(YamlNode node, WordsYaml doc, string path) + { + if (node is not YamlSequenceNode variants) + throw new WordsYamlException(path, "front matter.variants: expected a sequence"); + if (variants.Children.Count == 0) + throw new WordsYamlException(path, "front matter.variants: expected at least one named run"); + var names = new HashSet(StringComparer.Ordinal) { "default" }; + foreach (YamlNode variantNode in variants.Children) + { + if (variantNode is not YamlMappingNode variantMap) + throw new WordsYamlException(path, "each 'variants' entry must be a mapping"); + Dictionary fields = RequireMapping( + variantMap, + new HashSet(StringComparer.Ordinal) { "name", "morpher", "words" }, + path, + "variant" + ); + string name = RequireScalar(fields, "name", path, "variant"); + if (name.Any(c => !(c is >= 'a' and <= 'z' or >= '0' and <= '9' or '-'))) + throw new WordsYamlException(path, $"variant name '{name}' must match [a-z0-9-]+"); + if (!names.Add(name)) + throw new WordsYamlException(path, $"duplicate variant name '{name}' (default is reserved)"); + if (!fields.TryGetValue("morpher", out YamlNode morpherNode)) + throw new WordsYamlException(path, $"variant '{name}': missing required key 'morpher'"); + var settings = new MorpherSettings(); + ParseMorpher(morpherNode, settings, path); + if ( + !fields.TryGetValue("words", out YamlNode wordListNode) || wordListNode is not YamlSequenceNode wordList + ) + { + throw new WordsYamlException(path, $"variant '{name}': expected a 'words' sequence"); + } + List words = wordList.Children.Select(word => ParseWord(word, path)).ToList(); + if ( + words.Count != doc.Words.Count + || !words.Select(w => w.Word).SequenceEqual(doc.Words.Select(w => w.Word), StringComparer.Ordinal) + ) + { + throw new WordsYamlException( + path, + $"variant '{name}': words must contain exactly the default word list in the same order" + ); + } + foreach (WordEntry word in words) + { + if ( + word.BlockedBy.Count > 0 + || word.Neutralizes.Count > 0 + || word.Exercises.Count > 0 + || word.Claims.Count > 0 + || word.Parses.Any(p => p.Exercises.Count > 0) + ) + { + throw new WordsYamlException( + path, + $"variant '{name}', word '{word.Word}': coverage metadata is default-only" + ); + } + } + doc.Variants.Add(new WordRun(name, settings, words)); + } + } + + /// + /// Reads the morpher: mapping into . An empty block, an unknown key, + /// a non-integer or out-of-range count, or a non-boolean flag is an error: a setting the harness + /// cannot apply must not silently fall back to the default it was meant to override. + /// + private static void ParseMorpher(YamlNode node, MorpherSettings settings, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, "front matter.morpher: expected a mapping"); + Dictionary map = RequireMapping(mapNode, MorpherKeys, path, "front matter.morpher"); + if (map.Count == 0) + { + throw new WordsYamlException( + path, + "front matter.morpher: an empty block means the defaults; omit it instead" + ); + } + if (map.TryGetValue("DelReapps", out YamlNode delReappsNode)) + settings.DelReapps = ParseCount(delReappsNode, 0, path, "front matter.morpher.DelReapps"); + if (map.TryGetValue("MaxRoots", out YamlNode maxRootsNode)) + settings.MaxRoots = ParseCount(maxRootsNode, 1, path, "front matter.morpher.MaxRoots"); + if (map.TryGetValue("MergeAnalyses", out YamlNode mergeNode)) + { + settings.MergeAnalyses = ParseBool( + RequireScalarNode(mergeNode, path, "front matter.morpher.MergeAnalyses"), + path + ); + } + if (map.TryGetValue("MakePartialsFinal", out YamlNode partialsNode)) + { + settings.MakePartialsFinal = ParseBool( + RequireScalarNode(partialsNode, path, "front matter.morpher.MakePartialsFinal"), + path + ); + } + if (map.TryGetValue("GuessRoots", out YamlNode guessRootsNode)) + { + settings.GuessRoots = ParseBool( + RequireScalarNode(guessRootsNode, path, "front matter.morpher.GuessRoots"), + path + ); + } + } + + private static int ParseCount(YamlNode node, int minimum, string path, string context) + { + string text = RequireScalarNode(node, path, context); + if ( + !int.TryParse( + text, + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out int value + ) + || value < minimum + ) + { + throw new WordsYamlException( + path, + $"{context}: expected a whole number of at least {minimum}, got '{text}'" + ); + } + return value; + } + + private static WordEntry ParseWord(YamlNode node, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, "each 'words' entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, WordKeys, path, "word"); + + var entry = new WordEntry + { + Word = RequireScalar(map, "word", path, "word"), + Note = RequireScalar(map, "note", path, "word"), + }; + if (map.TryGetValue("provenance", out YamlNode provNode)) + entry.Provenance = RequireScalarNode(provNode, path, $"word '{entry.Word}'.provenance"); + if (map.TryGetValue("expect_fail", out YamlNode failNode)) + entry.ExpectFail = ParseBool(RequireScalarNode(failNode, path, $"word '{entry.Word}'.expect_fail"), path); + if (map.TryGetValue("expect_skip", out YamlNode skipNode)) + entry.ExpectSkip = ParseBool(RequireScalarNode(skipNode, path, $"word '{entry.Word}'.expect_skip"), path); + AppendScalarList(map, "blocked_by", entry.BlockedBy, path, $"word '{entry.Word}'"); + AppendScalarList(map, "neutralizes", entry.Neutralizes, path, $"word '{entry.Word}'"); + AppendScalarList(map, "exercises", entry.Exercises, path, $"word '{entry.Word}'"); + if (map.TryGetValue("claims", out YamlNode claimsNode)) + { + if (claimsNode is not YamlSequenceNode claimsSeq) + throw new WordsYamlException(path, $"word '{entry.Word}'.claims: expected a sequence"); + foreach (YamlNode claimNode in claimsSeq.Children) + entry.Claims.Add(ParseClaim(claimNode, entry.Word, path)); + } + + if (entry.ExpectFail && entry.ExpectSkip) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': 'expect_fail' and 'expect_skip' are mutually exclusive " + + "(a word is either a genuine zero-parse 'ok' or a SKIPPED InvalidShapeException, not both)" + ); + } + if (entry.BlockedBy.Count > 0 && !entry.ExpectFail) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': 'blocked_by' is only meaningful on an 'expect_fail' word" + ); + } + + bool hasParses = map.TryGetValue("parses", out YamlNode parsesNode); + if (entry.ExpectFail || entry.ExpectSkip) + { + if (hasParses) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': expect_fail/expect_skip words must not have 'parses'" + ); + } + } + else + { + if (!hasParses || parsesNode is not YamlSequenceNode parsesSeq || parsesSeq.Children.Count == 0) + { + throw new WordsYamlException( + path, + $"word '{entry.Word}': non-expect_fail words must have at least one entry under 'parses'" + ); + } + foreach (YamlNode parseNode in parsesSeq.Children) + entry.Parses.Add(ParseParse(parseNode, entry.Word, path)); + } + + return entry; + } + + private static ParseEntry ParseParse(YamlNode node, string word, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, $"word '{word}': each 'parses' entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, ParseKeys, path, $"word '{word}'.parses[]"); + + var parse = new ParseEntry { Signature = RequireScalar(map, "signature", path, $"word '{word}'.parses[]") }; + if (map.TryGetValue("gloss", out YamlNode glossNode)) + parse.Gloss = RequireScalarNode(glossNode, path, $"word '{word}'.parses[].gloss"); + if (map.TryGetValue("guess", out YamlNode guessNode)) + parse.Guess = ParseBool(RequireScalarNode(guessNode, path, $"word '{word}'.parses[].guess"), path); + if (!map.ContainsKey("rules")) + throw new WordsYamlException(path, $"word '{word}': every parse requires a 'rules' list (may be empty [])"); + AppendScalarList(map, "rules", parse.Rules, path, $"word '{word}'.parses[]"); + AppendScalarList(map, "exercises", parse.Exercises, path, $"word '{word}'.parses[]"); + + return parse; + } + + private static string ParseClaim(YamlNode node, string word, string path) + { + if (node is not YamlMappingNode mapNode) + throw new WordsYamlException(path, $"word '{word}'.claims[]: each entry must be a mapping"); + Dictionary map = RequireMapping(mapNode, ClaimKeys, path, $"word '{word}'.claims[]"); + return RequireScalar(map, "cell", path, $"word '{word}'.claims[]"); + } + + /// + /// Reads a mapping node's keys into a dictionary, hard-failing on any key not in + /// (the "unknown keys are hard errors" rule) and on any + /// non-scalar key (which would indicate a merge key or other non-plain construct). + /// + private static Dictionary RequireMapping( + YamlMappingNode node, + HashSet allowedKeys, + string path, + string context + ) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair entry in node.Children) + { + if (entry.Key is not YamlScalarNode keyScalar || keyScalar.Value == null) + throw new WordsYamlException(path, $"{context}: mapping keys must be plain scalars"); + string key = keyScalar.Value; + if (key == "<<") + throw new WordsYamlException(path, $"{context}: YAML merge keys ('<<') are not allowed"); + if (!allowedKeys.Contains(key)) + { + throw new WordsYamlException( + path, + $"{context}: unknown key '{key}' (allowed: {string.Join(", ", allowedKeys.OrderBy(k => k, StringComparer.Ordinal))})" + ); + } + if (map.ContainsKey(key)) + throw new WordsYamlException(path, $"{context}: duplicate key '{key}'"); + map[key] = entry.Value; + } + return map; + } + + private static string RequireScalar(Dictionary map, string key, string path, string context) + { + if (!map.TryGetValue(key, out YamlNode node)) + throw new WordsYamlException(path, $"{context}: missing required key '{key}'"); + return RequireScalarNode(node, path, $"{context}.{key}"); + } + + private static string RequireScalarNode(YamlNode node, string path, string context) + { + if (node is not YamlScalarNode scalar || scalar.Value == null || scalar.Value.Length == 0) + throw new WordsYamlException(path, $"{context}: expected a non-empty scalar value"); + return scalar.Value; + } + + private static void AppendScalarList( + Dictionary map, + string key, + List target, + string path, + string context + ) + { + if (!map.TryGetValue(key, out YamlNode node)) + return; + if (node is not YamlSequenceNode seq) + throw new WordsYamlException(path, $"{context}.{key}: expected a sequence"); + foreach (YamlNode child in seq.Children) + target.Add(RequireScalarNode(child, path, $"{context}.{key}[]")); + } + + private static bool ParseBool(string value, string path) + { + return value switch + { + "true" => true, + "false" => false, + _ => throw new WordsYamlException(path, $"expected 'true' or 'false', got '{value}'"), + }; + } +} + +public class WordsYamlException(string path, string message, Exception inner = null) + : Exception($"{path}: {message}", inner) { } diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs new file mode 100644 index 000000000..2835c8713 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/BatchCommand.cs @@ -0,0 +1,100 @@ +using System.Diagnostics; +using System.Globalization; +using System.IO; +using ManyConsole; +using Mono.Options; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Parses every word in a word list and records a per-word result signature plus elapsed time, so two +/// runs (for example, before and after an engine change) can be diffed to confirm parse results are +/// unchanged. Output is flushed per word, and --start resumes a run after a crash, because a single word +/// can take minutes. +/// +internal class BatchCommand : ConsoleCommand +{ + private const string UsageArguments = " "; + + private readonly HCContext _context; + private int _startIndex; + + public BatchCommand(HCContext context) + { + _context = context; + + IsCommand("batch", "Parses every word in a word list, recording a result signature and timing per word"); + SkipsCommandSummaryBeforeRunning(); + AllowsAnyAdditionalArguments(UsageArguments); + HasOption("start=", "0-based line index to resume at (for crash recovery)", v => _startIndex = ParseStart(v)); + } + + /// NumberStyles.None rejects a sign, so a negative index fails here rather than at words[i]. + private static int ParseStart(string value) + { + if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out int start)) + throw new OptionException($"--start must be a non-negative integer, got '{value}'", "--start"); + return start; + } + + /// + /// Counts the arguments here rather than with HasAdditionalArguments, which the dispatcher checks after + /// parsing --start. One instance serves every line of a script, so a line rejected there would leave + /// --start set for the next line. + /// + public override int? OverrideAfterHandlingArgumentsBeforeRun(string[] remainingArguments) + { + if (remainingArguments.Length == 2) + return null; + _startIndex = 0; + throw new ConsoleHelpAsException($"Invalid number of arguments - expected: {UsageArguments}"); + } + + public override int Run(string[] remainingArguments) + { + string wordListPath = remainingArguments[0]; + string outputPath = remainingArguments[1]; + + try + { + string[] words = SignatureFormat.LoadWords(wordListPath); + return RunSequential(words, outputPath); + } + finally + { + // One instance serves every line of a script, and --start is parsed only when given. + _startIndex = 0; + } + } + + private int RunSequential(string[] words, string outputPath) + { + using var writer = new StreamWriter(outputPath, append: _startIndex > 0) { AutoFlush = true }; + var totalSw = Stopwatch.StartNew(); + long parsed = 0, + skipped = 0; + for (int i = _startIndex; i < words.Length; i++) + { + string word = words[i]; + // Sentinel written before the attempt: if this word crashes the process, a wrapper script + // can read the last line to find where to resume. + writer.WriteLine($"{i}\t{word}\tSTARTED"); + (string status, long elapsedMs, string signature) = SignatureFormat.ParseOneWord(_context.Morpher, word); + writer.WriteLine($"{i}\t{word}\t{elapsedMs}\t{status}\t{signature}"); + if (status == "SKIPPED") + skipped++; + else + parsed++; + if (i % 100 == 0) + _context.Out.WriteLine("[{0}/{1}]", i, words.Length); + } + totalSw.Stop(); + _context.Out.WriteLine( + "batch complete: {0} words parsed ({1} skipped), {2}ms total", + parsed, + skipped, + totalSw.ElapsedMilliseconds + ); + return 0; + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs index ff8e86bcc..251e2f555 100644 --- a/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/Program.cs @@ -92,9 +92,11 @@ public static int Main(string[] args) new TracingCommand(context), new TestCommand(context), new StatsCommand(context), + new BatchCommand(context), }; string input; + int exitCode = 0; if (!string.IsNullOrEmpty(scriptFile)) { using (var scriptReader = new StreamReader(scriptFile)) @@ -105,7 +107,12 @@ public static int Main(string[] args) if (!input.Trim().StartsWith("#") && input.Trim() != "") { string[] cmdArgs = SplitCommandLine(input); - ConsoleCommandDispatcher.DispatchCommand(commands, cmdArgs, context.Out); + int result = ConsoleCommandDispatcher.DispatchCommand(commands, cmdArgs, context.Out); + // Remember the first nonzero result rather than the last: once a script + // line fails, later lines running (or not) shouldn't paper over that with + // a subsequent success. + if (result != 0 && exitCode == 0) + exitCode = result; } input = scriptReader.ReadLine(); } @@ -133,10 +140,13 @@ public static int Main(string[] args) output?.Close(); - return 0; + return exitCode; } - private static string[] SplitCommandLine(string commandLine) + /// Splits a command line into tokens, honoring double- and single-quoted segments. + /// Shared with SIL.Machine.Morphology.HermitCrab.Conformance's AdapterEngine via + /// InternalsVisibleTo so both consumers tokenize the exact same way. + internal static string[] SplitCommandLine(string commandLine) { var parmChars = commandLine.ToCharArray(); var inSingleQuote = false; diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..46d786444 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("hc-conformance")] +[assembly: InternalsVisibleTo("SIL.Machine.Morphology.HermitCrab.Tests")] diff --git a/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs b/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs new file mode 100644 index 000000000..27ee72505 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace SIL.Machine.Morphology.HermitCrab; + +internal static class SignatureFormat +{ + /// Reads a word list file: one word per line, trimmed, blank lines dropped. + public static string[] LoadWords(string path) + { + return File.ReadAllLines(path).Select(w => w.Trim()).Where(w => w.Length > 0).ToArray(); + } + + public static (string status, long elapsedMs, string signature) ParseOneWord(Morpher morpher, string word) + { + return ParseOneWord(morpher, word, guessRoot: false); + } + + public static (string status, long elapsedMs, string signature) ParseOneWord( + Morpher morpher, + string word, + bool guessRoot + ) + { + var sw = Stopwatch.StartNew(); + try + { + Word[] results = morpher.ParseWord(word, out _, guessRoot).ToArray(); + sw.Stop(); + return ("ok", sw.ElapsedMilliseconds, BuildSignature(results)); + } + catch (InvalidShapeException) + { + return ("SKIPPED", 0, "-"); + } + } + + /// + /// Builds the order-independent, sorted, semicolon-joined signature for a set of parse + /// results: one morph+morph|shape entry per distinct analysis, or "-" if empty. + /// + public static string BuildSignature(IEnumerable results) + { + List signatures = results + .Select(w => + string.Join("+", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Id)) + + "|" + + w.Shape.ToRegexString(w.Stratum.CharacterDefinitionTable, true) + ) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + return signatures.Count == 0 ? "-" : string.Join(";", signatures); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs new file mode 100644 index 000000000..dfcbdafb8 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/BatchCommandTests.cs @@ -0,0 +1,111 @@ +using ManyConsole; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class BatchCommandTests +{ + [Test] + public void Run_AfterAResumedRun_StartsTheNextRunAtTheFirstWord() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a", "b", "c"]); + try + { + string resumedPath = Path.Combine(directory, "resumed.tsv"); + string freshPath = Path.Combine(directory, "fresh.tsv"); + ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", "--start=2", wordsPath, resumedPath], + TextWriter.Null + ); + ConsoleCommandDispatcher.DispatchCommand(commands, ["batch", wordsPath, freshPath], TextWriter.Null); + + Assert.Multiple(() => + { + Assert.That(File.ReadAllLines(resumedPath)[0], Does.StartWith("2\t")); + Assert.That(File.ReadAllLines(freshPath)[0], Does.StartWith("0\t")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Test] + public void Run_AfterARejectedLineWithStart_StartsTheNextRunAtTheFirstWord() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a", "b", "c"]); + try + { + string freshPath = Path.Combine(directory, "fresh.tsv"); + // The output path is missing, so the dispatcher rejects the line after it has parsed --start. + int rejected = ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", "--start=2", wordsPath], + TextWriter.Null + ); + ConsoleCommandDispatcher.DispatchCommand(commands, ["batch", wordsPath, freshPath], TextWriter.Null); + + Assert.Multiple(() => + { + Assert.That(rejected, Is.Not.EqualTo(0)); + Assert.That(File.ReadAllLines(freshPath)[0], Does.StartWith("0\t")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [TestCase("-1")] + [TestCase("x")] + [TestCase("+1")] + public void Run_WithAnInvalidStart_ReportsItWithoutThrowing(string start) + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + var context = new HCContext(XmlLanguageLoader.Load(fixture.GrammarPath), TextWriter.Null); + context.Compile(); + ConsoleCommand[] commands = [new BatchCommand(context)]; + string directory = Path.Combine(Path.GetTempPath(), $"batch-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string wordsPath = Path.Combine(directory, "words.txt"); + File.WriteAllLines(wordsPath, ["a"]); + try + { + var console = new StringWriter(); + int result = ConsoleCommandDispatcher.DispatchCommand( + commands, + ["batch", $"--start={start}", wordsPath, Path.Combine(directory, "out.tsv")], + console + ); + + Assert.Multiple(() => + { + Assert.That(result, Is.Not.EqualTo(0)); + Assert.That(console.ToString(), Does.Contain("--start must be a non-negative integer")); + }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs new file mode 100644 index 000000000..1c5a163e5 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceCommandLineTests.cs @@ -0,0 +1,144 @@ +using System.Reflection; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceCommandLineTests +{ + private static (int ExitCode, string Output, string Error) InvokeProgram(params string[] args) + { + Type programType = typeof(Runner).Assembly.GetType("SIL.Machine.Morphology.HermitCrab.Conformance.Program")!; + MethodInfo main = programType.GetMethod("Main", BindingFlags.Static | BindingFlags.NonPublic)!; + TextWriter originalOut = Console.Out; + TextWriter originalError = Console.Error; + using var output = new StringWriter(); + using var error = new StringWriter(); + try + { + Console.SetOut(output); + Console.SetError(error); + + int exitCode = (int)main.Invoke(null, new object[] { args })!; + return (exitCode, output.ToString(), error.ToString()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + [Test] + [NonParallelizable] + public void CoverageReport_MissingConstructChecklist_Fails() + { + string missing = Path.Combine(Path.GetTempPath(), $"constructs-{Guid.NewGuid():N}.txt"); + + (int exitCode, string output, string error) = InvokeProgram( + "--fixtures", + ConformanceSuite.Root(), + "--coverage-report", + "--constructs", + missing + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(1), error); + Assert.That(output, Does.Contain("CONSTRUCT CHECKLIST NOT FOUND")); + }); + } + + [Test] + [NonParallelizable] + public void CoverageReport_WritesNothingUnderTheFixturesRoot() + { + string suite = ConformanceSuite.Root(); + string[] files = { Path.Combine(suite, "coverage.csv"), Path.Combine(suite, "rules.csv") }; + DateTime[] before = files.Select(File.GetLastWriteTimeUtc).ToArray(); + string[] entriesBefore = Directory.GetFileSystemEntries(suite); + string output = Path.Combine(Path.GetTempPath(), $"hc-coverage-out-{Guid.NewGuid():N}"); + try + { + (int exitCode, string text, string error) = InvokeProgram( + "--fixtures", + suite, + "--coverage-report", + "--coverage-output", + output + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(0), text + error); + Assert.That(files.Select(File.GetLastWriteTimeUtc), Is.EqualTo(before)); + Assert.That(Directory.GetFileSystemEntries(suite), Is.EquivalentTo(entriesBefore)); + foreach (string file in files) + { + Assert.That( + File.ReadAllText(Path.Combine(output, Path.GetFileName(file))).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(file).ReplaceLineEndings("\n")), + Path.GetFileName(file) + ); + } + }); + } + finally + { + if (Directory.Exists(output)) + Directory.Delete(output, recursive: true); + } + } + + [Test] + [NonParallelizable] + public void CoverageReport_RefusesAnOutputInsideTheFixturesRoot() + { + string suite = ConformanceSuite.Root(); + string inside = Path.Combine(suite, $"hc-coverage-out-{Guid.NewGuid():N}"); + + (int exitCode, string _, string error) = InvokeProgram( + "--fixtures", + suite, + "--coverage-report", + "--coverage-output", + inside + ); + + Assert.Multiple(() => + { + Assert.That(exitCode, Is.EqualTo(2), error); + Assert.That(error, Does.Contain("not the suite clone")); + Assert.That(Directory.Exists(inside), Is.False); + }); + } + + [Test] + [NonParallelizable] + public void CheckManifest_MalformedGrammar_NamesTheFixtureAndFails() + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = ConformanceManifestTests.CreateMiniSuite(FixtureId); + try + { + File.WriteAllText( + Path.Combine(root, "edge-cases", "strrep-identity", "grammar.xml"), + " + { + Assert.That(exitCode, Is.EqualTo(2), error); + Assert.That(error, Does.Contain(FixtureId)); + }); + } + finally + { + Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs new file mode 100644 index 000000000..3eb24f37c --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceFixtureGateTests.cs @@ -0,0 +1,353 @@ +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Runs every conformance fixture inside the ordinary test suite, so a change to the engine, the runner or +/// the suite pin that breaks a fixture fails dotnet test as well as the standalone hc-conformance CLI. +/// +[TestFixture] +public sealed class ConformanceFixtureGateTests +{ + /// Guards against the suite silently shrinking to nothing and still reporting green. + private const int MinimumFixtures = 25; + + private static List Discover() => Fixture.DiscoverAll(ConformanceSuite.Root()); + + [Test] + public void EveryConformanceFixturePassesSelfCheck() + { + List fixtures = Discover(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(MinimumFixtures)); + + RunReport report = Runner.RunSelfCheck( + fixtures, + includePathological: false, + Runner.DefaultCapabilities, + propose: false, + TextWriter.Null + ); + + string[] failures = report + .Results.Where(result => result.Outcome == FixtureOutcome.Failed) + .Select(result => $"{result.FixtureId}: {result.Reason}") + .OrderBy(line => line, StringComparer.Ordinal) + .ToArray(); + + Assert.That(failures, Is.Empty, $"conformance fixtures failed:\n {string.Join("\n ", failures)}"); + Assert.That( + report.Passed, + Is.GreaterThanOrEqualTo(MinimumFixtures), + "the run must actually execute fixtures, not skip them all" + ); + } + + [Test] + public void EveryBlockedByLabelIsARuleIdInItsFixtureGrammar() + { + List fixtures = Discover(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(MinimumFixtures)); + + var unknown = new List(); + foreach (Fixture fixture in fixtures) + { + GrammarRuleIndex index = GrammarRuleIndex.Load(fixture.GrammarPath); + unknown.AddRange(CoverageReport.FindUnknownBlockedBy(fixture.Id, fixture.Words, index.AllRuleIds)); + } + + Assert.That( + unknown.Select(u => $"{u.FixtureId}: {u.Word} blocked_by {u.Label}"), + Is.Empty, + "blocked_by labels must name a rule id declared in the fixture's grammar.xml" + ); + } + + [Test] + public void CheckedInCoverageTablesAreUpToDate() + { + string root = ConformanceSuite.Root(); + string temp = Path.Combine(Path.GetTempPath(), $"hc-coverage-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temp); + try + { + string coverage = Path.Combine(temp, "coverage.csv"); + string rules = Path.Combine(temp, "rules.csv"); + CoverageReport.WriteCsvs(Discover(), coverage, rules); + + Assert.Multiple(() => + { + Assert.That( + File.ReadAllText(coverage).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(Path.Combine(root, "coverage.csv")).ReplaceLineEndings("\n")), + "regenerate in a clone of the suite with: hc-conformance --fixtures conformance --coverage-report --coverage-output " + ); + Assert.That( + File.ReadAllText(rules).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(Path.Combine(root, "rules.csv")).ReplaceLineEndings("\n")), + "regenerate in a clone of the suite with: hc-conformance --fixtures conformance --coverage-report --coverage-output " + ); + }); + } + finally + { + Directory.Delete(temp, recursive: true); + } + } + + [Test] + public void EveryFixtureContributesAtLeastOneCoverageRow() + { + var noExerciseByDesign = new HashSet(StringComparer.Ordinal) + { + "edge-cases/simultaneous-epenthesis-cascade", + "edge-cases/truncate-morphotactic-copy-u", + }; + + List fixtures = Discover(); + Assert.That( + fixtures, + Has.Count.GreaterThanOrEqualTo(MinimumFixtures), + "the discovery walk must find fixtures, not silently see zero" + ); + + HashSet discoveredIds = fixtures.Select(f => f.Id).ToHashSet(StringComparer.Ordinal); + string[] staleAllowlistEntries = noExerciseByDesign + .Where(id => !discoveredIds.Contains(id)) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + Assert.That( + staleAllowlistEntries, + Is.Empty, + "these allowlist entries name no discovered fixture (renamed or typo'd?):\n " + + string.Join("\n ", staleAllowlistEntries) + ); + + string temp = Path.Combine(Path.GetTempPath(), $"hc-exercise-gate-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temp); + try + { + string coverage = Path.Combine(temp, "coverage.csv"); + string rules = Path.Combine(temp, "rules.csv"); + CoverageReport.WriteCsvs(fixtures, coverage, rules); + + string[] lines = File.ReadAllLines(coverage); + Assert.That( + lines, + Has.Length.GreaterThan(1), + "coverage.csv must have a header plus at least one data row -- an empty table proves nothing" + ); + + HashSet languagesWithRows = lines + .Skip(1) + .Select(line => line[..line.IndexOf(',')]) + .ToHashSet(StringComparer.Ordinal); + + string[] silent = fixtures + .Where(f => !languagesWithRows.Contains(f.Words.Language) && !noExerciseByDesign.Contains(f.Id)) + .Select(f => f.Id) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + silent, + Is.Empty, + "these fixtures contribute ZERO rows to coverage.csv (no 'exercises:' tag anywhere) and " + + $"are not in the named allowlist above:\n {string.Join("\n ", silent)}" + ); + } + finally + { + Directory.Delete(temp, recursive: true); + } + } + + // Every construct a fixture claims must be a real line in the checklist, and every checklist line + // must be claimed. parity-check.py enforces the second half; nothing enforced the first. + [Test] + public void EveryExercisedConstructIsDeclaredInTheChecklist() + { + string root = ConformanceSuite.Root(); + var declared = CoverageReport + .LoadConstructChecklist(Path.Combine(root, "constructs.txt")) + .ToHashSet(StringComparer.Ordinal); + + string[] unknown = Discover() + .SelectMany(fixture => fixture.Words.Words) + .SelectMany(word => word.Exercises.Concat(word.Parses.SelectMany(parse => parse.Exercises))) + .Distinct(StringComparer.Ordinal) + .Where(construct => !declared.Contains(construct)) + .OrderBy(construct => construct, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + unknown, + Is.Empty, + $"add these to constructs.txt or fix the tag:\n {string.Join("\n ", unknown)}" + ); + } + + [Test] + public void FixturesWithRulesAttributeAtLeastOneParseToThem() + { + var silent = new List(); + foreach (Fixture fixture in Discover()) + { + bool declaresRules = File.ReadAllText(fixture.GrammarPath) + .Contains("MorphologicalRule id=", StringComparison.Ordinal); + bool attributesAny = fixture.Words.Words.Any(word => word.Parses.Any(parse => parse.Rules.Count > 0)); + if (declaresRules && !attributesAny) + silent.Add(fixture.Id); + } + + Assert.That( + silent, + Is.Empty, + "these declare morphological rules but no parse names one, so nothing they contain can be trace-verified" + ); + } + + [Test] + public void EveryFixtureIsMarkedFieldWorksProducible() + { + string[] notProducible = Discover() + .Where(fixture => fixture.Words.FieldworksProducible != true) + .Select(fixture => fixture.Id) + .ToArray(); + + Assert.That( + notProducible, + Is.Empty, + "conformance keeps only grammars a FieldWorks project can produce (PROTOCOL.md section 9); rewrite or delete these" + ); + } + + [Test] + public void FieldWorksWitnessDirectoriesAreShapedCorrectly() + { + var allowedManifestKeys = new HashSet(StringComparer.Ordinal) + { + "version", + "base_sha256", + "cases", + "id", + "operations", + "expect", + "remove_phoneme", + "remove_all_phonemes", + "guid", + "assert_representations", + "require_unreferenced", + "xample_projection", + "hc_analyses", + "inferred_segments", + }; + + var errors = new List(); + foreach (Fixture fixture in Discover()) + { + string fieldworksDir = Path.Combine(fixture.Directory, "fieldworks"); + if (!Directory.Exists(fieldworksDir)) + continue; + + if (fixture.Words.FieldworksProducible != true) + { + errors.Add($"{fixture.Id}: has a fieldworks/ directory but is not 'fieldworks_producible: true'"); + continue; + } + + var expectedEntries = new HashSet(StringComparer.Ordinal) + { + "project.fwdata", + "phonology-mutations.yaml", + "WritingSystemStore", + }; + string[] actualEntries = Directory.GetFileSystemEntries(fieldworksDir).Select(Path.GetFileName).ToArray()!; + foreach (string extra in actualEntries.Where(e => !expectedEntries.Contains(e!))) + errors.Add($"{fixture.Id}: fieldworks/ contains an unexpected entry '{extra}'"); + foreach (string missing in expectedEntries.Where(e => !actualEntries.Contains(e))) + errors.Add($"{fixture.Id}: fieldworks/ is missing '{missing}'"); + + string writingSystemDir = Path.Combine(fieldworksDir, "WritingSystemStore"); + if (Directory.Exists(writingSystemDir)) + { + string[] nonLdml = Directory + .GetFiles(writingSystemDir) + .Select(Path.GetFileName) + .Where(name => !name!.EndsWith(".ldml", StringComparison.OrdinalIgnoreCase)) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray()!; + if (nonLdml.Length > 0) + { + errors.Add( + $"{fixture.Id}: WritingSystemStore/ contains non-.ldml entries: {string.Join(", ", nonLdml)}" + ); + } + } + + string projectPath = Path.Combine(fieldworksDir, "project.fwdata"); + string manifestPath = Path.Combine(fieldworksDir, "phonology-mutations.yaml"); + if (!File.Exists(projectPath) || !File.Exists(manifestPath)) + continue; // already recorded as missing above + + string manifestText = File.ReadAllText(manifestPath); + + Match versionMatch = Regex.Match(manifestText, @"^version:\s*(\d+)\s*$", RegexOptions.Multiline); + if (!versionMatch.Success || versionMatch.Groups[1].Value != "1") + errors.Add($"{fixture.Id}: phonology-mutations.yaml has no 'version: 1'"); + + Match shaMatch = Regex.Match(manifestText, @"^base_sha256:\s*([0-9a-f]{64})\s*$", RegexOptions.Multiline); + if (!shaMatch.Success) + { + errors.Add($"{fixture.Id}: phonology-mutations.yaml has no 64-character lowercase-hex 'base_sha256'"); + } + else + { + using var sha256 = SHA256.Create(); + using FileStream stream = File.OpenRead(projectPath); + string actual = Convert.ToHexString(sha256.ComputeHash(stream)).ToLowerInvariant(); + if (!string.Equals(actual, shaMatch.Groups[1].Value, StringComparison.Ordinal)) + { + errors.Add( + $"{fixture.Id}: phonology-mutations.yaml base_sha256 {shaMatch.Groups[1].Value} " + + $"does not match project.fwdata's actual sha256 {actual}" + ); + } + } + + foreach ( + Match keyMatch in Regex.Matches( + manifestText, + @"^\s*-?\s*([A-Za-z_][A-Za-z0-9_]*):", + RegexOptions.Multiline + ) + ) + { + string key = keyMatch.Groups[1].Value; + if (!allowedManifestKeys.Contains(key)) + errors.Add($"{fixture.Id}: phonology-mutations.yaml uses a key outside the v1 vocabulary: '{key}'"); + } + + foreach ( + Match valueMatch in Regex.Matches( + manifestText, + @"(xample_projection|hc_analyses):\s*(\S+)\s*$", + RegexOptions.Multiline + ) + ) + { + if (valueMatch.Groups[2].Value != "same_as_base") + { + errors.Add( + $"{fixture.Id}: phonology-mutations.yaml has {valueMatch.Groups[1].Value}: " + + $"{valueMatch.Groups[2].Value}, but only 'same_as_base' is defined" + ); + } + } + } + + Assert.That(errors, Is.Empty, string.Join("\n ", errors)); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs new file mode 100644 index 000000000..aa4624125 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceHarnessTests.cs @@ -0,0 +1,248 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceHarnessTests +{ + private sealed class AlwaysCrashingEngine(IEnumerable capabilities) : IEngine + { + public string Name => "always-crashing"; + public IReadOnlySet Capabilities { get; } = new HashSet(capabilities, StringComparer.Ordinal); + + public List Run(MaterializedFixture fixture) => + throw new EngineCrashException($"stub crash for '{fixture.Id}'"); + } + + // A copy of the slow fixture with a 1 ms budget, so the self-check watchdog fires after its 1 s margin while + // the fast first word has already finished and the 924-parse second word has not. + private static FixtureResult RunSlowFixtureOverBudget(string extraFrontMatter) + { + string root = Path.Combine(Path.GetTempPath(), $"slow-fixture-{Guid.NewGuid():N}"); + string source = Path.Combine(ConformanceSuite.Root(), "edge-cases", "deep-optional-affix-nesting"); + string target = Path.Combine(root, "edge-cases", "deep-optional-affix-nesting"); + Directory.CreateDirectory(target); + foreach (string file in Directory.GetFiles(source)) + File.Copy(file, Path.Combine(target, Path.GetFileName(file))); + string wordsPath = Path.Combine(target, "words.yaml"); + string words = File.ReadAllText(wordsPath); + Assert.That(words, Does.Contain("budget_ms: 15000")); + File.WriteAllText(wordsPath, words.Replace("budget_ms: 15000", "budget_ms: 1" + extraFrontMatter)); + try + { + RunReport report = Runner.RunSelfCheck( + Fixture.DiscoverAll(root), + includePathological: true, + engineCapabilities: new HashSet(StringComparer.Ordinal), + propose: false, + proposeOutput: TextWriter.Null + ); + return report.Results.Single(); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + private static List OneRow(string signature) => [new TsvRow("bat", "ok", signature)]; + + [Test] + public void Diff_ClassMembersInDifferentOrder_Passes() + { + FixtureDiff diff = Diff.Compare(OneRow("|b[ae]t"), OneRow("|b[ea]t")); + + Assert.That(diff.AllPassed, Is.True); + } + + [Test] + public void Diff_DifferentClassMembers_Fails() + { + FixtureDiff diff = Diff.Compare(OneRow("|b[ae]t"), OneRow("|b[ao]t")); + + Assert.That(diff.AllPassed, Is.False); + } + + [Test] + public void Diff_NegatedClassMembersInDifferentOrder_PassesButNegationStillMatters() + { + Assert.That(Diff.Compare(OneRow("|b[^ae]t"), OneRow("|b[^ea]t")).AllPassed, Is.True); + Assert.That(Diff.Compare(OneRow("|b[^ae]t"), OneRow("|b[ae]t")).AllPassed, Is.False); + } + + [Test] + public void CompareRuleTraceMultisets_SameSignatureWithDistinctTraces_PassesRegardlessOfOrder() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead", "crShared" }), + (Signature, new[] { "crRightHead" }), + }; + var actual = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crRightHead" }), + (Signature, new[] { "crShared", "crLeftHead" }), + }; + + Assert.That(Runner.CompareRuleTraceMultisets(expected, actual), Is.Empty); + } + + [Test] + public void CompareRuleTraceMultisets_WrongTraceForSameSignature_Fails() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] { (Signature, new[] { "crLeftHead" }) }; + var actual = new (string Signature, IEnumerable RuleIds)[] { (Signature, new[] { "crWrongHead" }) }; + + List details = Runner.CompareRuleTraceMultisets(expected, actual); + + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details, Has.Some.Contains("rules [crLeftHead]: declared 1, traced 0")); + Assert.That(details, Has.Some.Contains("rules [crWrongHead]: declared 0, traced 1")); + } + + [Test] + public void CompareRuleTraceMultisets_DuplicateTraceMultiplicityMustMatch() + { + const string Signature = "DAK+IMO|dakimo"; + var expected = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead" }), + (Signature, new[] { "crLeftHead" }), + }; + var actual = new (string Signature, IEnumerable RuleIds)[] + { + (Signature, new[] { "crLeftHead" }), + (Signature, new[] { "crRightHead" }), + }; + + List details = Runner.CompareRuleTraceMultisets(expected, actual); + + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details, Has.Some.Contains("rules [crLeftHead]: declared 2, traced 1")); + Assert.That(details, Has.Some.Contains("rules [crRightHead]: declared 0, traced 1")); + } + + [Test] + public void CompareRuleTraceMultisets_BracketClassMembersInDifferentOrder_Passes() + { + var expected = new (string Signature, IEnumerable RuleIds)[] { ("|b[ea]t", new[] { "crLeftHead" }) }; + var actual = new (string Signature, IEnumerable RuleIds)[] { ("|b[ae]t", new[] { "crLeftHead" }) }; + + Assert.That(Runner.CompareRuleTraceMultisets(expected, actual), Is.Empty); + } + + [Test] + public void RunSelfCheck_ExpectCrashFixtureThatTimesOut_Fails() + { + FixtureResult result = RunSlowFixtureOverBudget("\nexpect_crash: true"); + + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + } + + [Test] + public void RunSelfCheck_TimedOutFixture_ReportsNoWordResults() + { + FixtureResult result = RunSlowFixtureOverBudget(""); + + Assert.Multiple(() => + { + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + Assert.That(result.WordResults, Is.Empty); + }); + } + + [Test] + public void RunSelfCheck_ExpectCrashFixtureWhoseGrammarTheHarnessRejects_Fails() + { + string root = Path.Combine(Path.GetTempPath(), $"harness-failure-{Guid.NewGuid():N}"); + string source = Path.Combine(ConformanceSuite.Root(), "edge-cases", "epenthesis-fed-merge"); + string target = Path.Combine(root, "edge-cases", "epenthesis-fed-merge"); + Directory.CreateDirectory(target); + foreach (string file in Directory.GetFiles(source)) + File.Copy(file, Path.Combine(target, Path.GetFileName(file))); + string wordsPath = Path.Combine(target, "words.yaml"); + string words = File.ReadAllText(wordsPath); + Assert.That(words, Does.Contain("\nwords:\n")); + File.WriteAllText(wordsPath, words.Replace("\nwords:\n", "\nexpect_crash: true\nwords:\n")); + // The engine loads this grammar, but the harness's rule index rejects two rules with one name. + string grammarPath = Path.Combine(target, "grammar.xml"); + string grammar = File.ReadAllText(grammarPath); + Assert.That(grammar, Does.Contain("mergeTi")); + File.WriteAllText(grammarPath, grammar.Replace("mergeTi", "insertTiAfterBAtEnd")); + try + { + RunReport report = Runner.RunSelfCheck( + Fixture.DiscoverAll(root), + includePathological: true, + engineCapabilities: Runner.DefaultCapabilities, + propose: false, + proposeOutput: TextWriter.Null + ); + + FixtureResult result = report.Results.Single(); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public void RunAdapter_ExpectCrashFixture_FailsWhenTheEngineAlsoCrashesOnAnOrdinaryFixture() + { + List all = Fixture.DiscoverAll(ConformanceSuite.Root()); + Fixture crash = all.First(fixture => fixture.Words.ExpectCrash); + Fixture ordinary = all.First(fixture => !fixture.Words.ExpectCrash && fixture.Words.BudgetMs is null); + var engine = new AlwaysCrashingEngine(crash.Words.Requires.Concat(ordinary.Words.Requires)); + + RunReport report = Runner.RunAdapter(new List { crash, ordinary }, engine, includePathological: true); + + FixtureResult crashResult = report.Results.Single(result => result.FixtureId == crash.Id); + Assert.That(crashResult.Outcome, Is.EqualTo(FixtureOutcome.Failed), crashResult.Reason); + Assert.That(crashResult.Reason, Does.Contain(ordinary.Id)); + } + + [Test] + public void WordsYamlLoader_NonNumericBudget_ThrowsWordsYamlException() + { + string path = Path.Combine(Path.GetTempPath(), $"words-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(path, "language: Probe\nbudget_ms: 5s\nwords: []\n"); + try + { + WordsYamlException ex = Assert.Throws(() => WordsYamlLoader.Load(path))!; + Assert.That(ex.Message, Does.Contain("budget_ms")); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void GrammarRuleIndex_TwoRulesWithTheSameName_Throws() + { + string path = Path.Combine(Path.GetTempPath(), $"grammar-{Guid.NewGuid():N}.xml"); + File.WriteAllText( + path, + """ + + Epenthesis + Epenthesis + + """ + ); + try + { + InvalidDataException ex = Assert.Throws(() => GrammarRuleIndex.Load(path))!; + Assert.That(ex.Message, Does.Contain("pr1").And.Contain("pr2")); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs new file mode 100644 index 000000000..5347421e6 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceManifestTests.cs @@ -0,0 +1,302 @@ +using System.Globalization; +using System.Text.Json; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceManifestTests +{ + private static string SuiteRoot() => ConformanceSuite.Root(); + + private static ConformanceManifest Manifest() => ConformanceManifestGenerator.Generate(SuiteRoot()); + + /// A path as the manifest names it: relative to the suite root, the conformance suite. + private static string InSuite(string manifestPath) => + Path.Combine(SuiteRoot(), manifestPath.Replace('/', Path.DirectorySeparatorChar)); + + private const string LibraryDtdRelativePath = "src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd"; + + private static string LibraryDtdPath => + Path.Combine( + ConformanceSuite.RepositoryRoot(), + LibraryDtdRelativePath.Replace('/', Path.DirectorySeparatorChar) + ); + + [Test] + public void FormatIdentifiersAreExact() + { + ConformanceManifest manifest = Manifest(); + + Assert.Multiple(() => + { + Assert.That(manifest.FormatVersion, Is.EqualTo("hc-conformance-manifest/v2")); + Assert.That(manifest.WordsAuthoringFormatVersion, Is.EqualTo("hc-conformance-words/v1")); + Assert.That(manifest.GrammarFormatVersion, Is.EqualTo("sil-machine-hermit-crab-input-xml/v1")); + Assert.That(manifest.SourceHash, Has.Length.EqualTo(64)); + Assert.That(manifest.DtdSha256, Has.Length.EqualTo(64)); + Assert.That(manifest.Fixtures, Is.Not.Empty); + }); + } + + // The suite is consumed on its own, so every path the manifest names is relative to the suite root + // and carries no directory prefix. + [Test] + public void EveryPathTheManifestNamesIsRelativeToTheSuiteRoot() + { + ConformanceManifest manifest = Manifest(); + + Assert.Multiple(() => + { + Assert.That(manifest.DtdPath, Is.EqualTo("HermitCrabInput.dtd")); + Assert.That(File.Exists(InSuite(manifest.DtdPath)), Is.True, manifest.DtdPath); + foreach (ManifestFixture fixture in manifest.Fixtures) + { + Assert.That(fixture.GrammarPath, Is.EqualTo($"{fixture.FixtureId}/grammar.xml")); + Assert.That(fixture.WordsPath, Is.EqualTo($"{fixture.FixtureId}/words.yaml")); + Assert.That(File.Exists(InSuite(fixture.GrammarPath)), Is.True, fixture.GrammarPath); + Assert.That(File.Exists(InSuite(fixture.WordsPath)), Is.True, fixture.WordsPath); + Assert.That(fixture.GrammarPath, Does.Not.Contain("\\")); + Assert.That(fixture.GrammarSha256, Has.Length.EqualTo(64)); + Assert.That(fixture.WordsSha256, Has.Length.EqualTo(64)); + Assert.That(fixture.DisplayLanguage, Is.Not.Empty, fixture.FixtureId); + Assert.That(fixture.CaseCount, Is.GreaterThan(0), fixture.FixtureId); + Assert.That( + fixture.Category, + Is.EqualTo( + fixture.FixtureId.StartsWith("languages/", StringComparison.Ordinal) ? "language" : "edge-case" + ) + ); + } + }); + } + + // The library reads its DTD as an embedded resource and cannot move; the suite's copy is the + // authority. They must not drift. + [Test] + public void PublishedDtdMatchesTheLibraryResource() + { + Assert.That( + File.ReadAllText(InSuite(ConformanceManifestGenerator.DtdRelativePath)).ReplaceLineEndings("\n"), + Is.EqualTo(File.ReadAllText(LibraryDtdPath).ReplaceLineEndings("\n")), + $"the suite's {ConformanceManifestGenerator.DtdRelativePath} is authoritative: change it in the suite, bump " + + $"conformance.commit and copy it over {LibraryDtdRelativePath}" + ); + } + + [Test] + public void CrashFixturesDeclareExactlyOneCase() + { + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures.Where(item => item.ExpectedCrash)) + Assert.That(fixture.CaseCount, Is.EqualTo(1), fixture.FixtureId); + }); + } + + // A regenerated manifest must be byte-identical, or `--check-manifest` cannot detect drift. + [Test] + public void GenerationIsDeterministicAndCanonical() + { + string first = ManifestJson.Serialize(Manifest()); + string second = ManifestJson.Serialize(Manifest()); + + Assert.That(second, Is.EqualTo(first)); + Assert.That(() => JsonDocument.Parse(first), Throws.Nothing); + } + + [Test] + public void TheCheckedInManifestMatchesRegeneration() + { + string path = InSuite(ConformanceManifestGenerator.ManifestRelativePath); + const string Fix = "regenerate it in the suite with: python3 scripts/generate_manifest.py"; + + Assert.That(File.Exists(path), Is.True, Fix); + Assert.That( + File.ReadAllText(path).ReplaceLineEndings("\n"), + Is.EqualTo(ManifestJson.Serialize(Manifest()) + "\n"), + Fix + ); + } + + [Test] + public void TheManifestValidatesAgainstItsPublishedSchema() + { + Json.Schema.JsonSchema schema = Json.Schema.JsonSchema.FromFile( + InSuite("schema/conformance-manifest.schema.json") + ); + using JsonDocument document = JsonDocument.Parse(ManifestJson.Serialize(Manifest())); + + Json.Schema.EvaluationResults results = schema.Evaluate( + document.RootElement, + new Json.Schema.EvaluationOptions { OutputFormat = Json.Schema.OutputFormat.List } + ); + + IEnumerable failures = (results.Details ?? new List()) + .Where(detail => detail.Errors is { Count: > 0 }) + .SelectMany(detail => detail.Errors!.Select(error => $"{detail.InstanceLocation}: {error.Value}")) + .Take(20); + + Assert.That(results.IsValid, Is.True, string.Join(Environment.NewLine, failures)); + } + + // words.yaml is the canonical authored format, so the published schema has to describe every + // file the loader already accepts. + [Test] + public void EveryWordsFileValidatesAgainstThePublishedSchema() + { + string schema = InSuite(WordsSchemaValidation.SchemaRelativePath); + + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures) + { + IReadOnlyList violations = WordsSchemaValidation.Validate(InSuite(fixture.WordsPath), schema); + Assert.That(violations, Is.Empty, $"{fixture.FixtureId}: {string.Join(" | ", violations.Take(6))}"); + } + }); + } + + [TestCase("anchors", "language: X\nwords: &a []\n")] + [TestCase("merge keys", "language: X\nwords:\n - <<: {word: a}\n note: n\n")] + public void PlainYamlOnlyIsEnforced(string _, string yaml) + { + string schema = InSuite(WordsSchemaValidation.SchemaRelativePath); + string path = Path.Combine(Path.GetTempPath(), $"hc-words-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(path, yaml); + try + { + Assert.That(() => WordsSchemaValidation.Validate(path, schema), Throws.TypeOf()); + } + finally + { + File.Delete(path); + } + } + + /// A minimal suite carrying one real fixture, for mutation tests. + internal static string CreateMiniSuite(string fixtureId) + { + string source = SuiteRoot(); + string root = Path.Combine(Path.GetTempPath(), $"hc-manifest-suite-{Guid.NewGuid():N}"); + + void Copy(string relative) + { + string target = Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(Path.Combine(source, relative.Replace('/', Path.DirectorySeparatorChar)), target); + } + + Copy(ConformanceManifestGenerator.DtdRelativePath); + Copy("constructs.txt"); + Copy("parity-check.py"); + Copy("schema/words.schema.json"); + Copy("schema/conformance-manifest.schema.json"); + Copy($"{fixtureId}/grammar.xml"); + Copy($"{fixtureId}/words.yaml"); + return root; + } + + // Every input the manifest rests on must move its sourceHash, or a stale manifest cannot be + // told from a current one. + [TestCase("{0}/grammar.xml")] + [TestCase("{0}/words.yaml")] + [TestCase("constructs.txt")] + [TestCase("parity-check.py")] + [TestCase("schema/words.schema.json")] + public void EverySourceFileMovesTheSourceHash(string relativeTemplate) + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = CreateMiniSuite(FixtureId); + try + { + string before = ConformanceManifestGenerator.Generate(root).SourceHash; + string target = Path.Combine( + root, + string.Format(CultureInfo.InvariantCulture, relativeTemplate, FixtureId) + .Replace('/', Path.DirectorySeparatorChar) + ); + File.AppendAllText(target, "\n# drift\n"); + + Assert.That( + ConformanceManifestGenerator.Generate(root).SourceHash, + Is.Not.EqualTo(before), + relativeTemplate + ); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + // Fixture identity comes from the two declared roots only; a stray directory elsewhere under + // the suite root must not be able to present itself as a fixture. + [Test] + public void DiscoveryAcceptsOnlyTheTwoDeclaredFixtureRoots() + { + const string FixtureId = "edge-cases/strrep-identity"; + string root = CreateMiniSuite(FixtureId); + try + { + string legacy = Path.Combine(root, "allomorphy", "strrep-identity"); + Directory.CreateDirectory(legacy); + foreach (string name in new[] { "grammar.xml", "words.yaml" }) + { + File.Copy(Path.Combine(root, "edge-cases", "strrep-identity", name), Path.Combine(legacy, name)); + } + + Assert.That( + ConformanceManifestGenerator.Generate(root).Fixtures.Select(fixture => fixture.FixtureId), + Is.EqualTo(new[] { FixtureId }) + ); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public void EveryGrammarValidatesAgainstThePublishedDtd() + { + string dtd = InSuite(ConformanceManifestGenerator.DtdRelativePath); + + Assert.Multiple(() => + { + foreach (ManifestFixture fixture in Manifest().Fixtures) + { + Assert.That( + GrammarValidation.Validate(InSuite(fixture.GrammarPath), dtd), + Is.Empty, + $"{fixture.FixtureId}: grammar must validate against the published DTD" + ); + } + }); + } + + [Test] + public void TheResolverAdmitsOnlyThePinnedDtdSystemIdentifier() + { + string dtd = InSuite(ConformanceManifestGenerator.DtdRelativePath); + string directory = Path.Combine(Path.GetTempPath(), $"hc-manifest-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + string hostile = Path.Combine(directory, "grammar.xml"); + File.WriteAllText( + hostile, + "\n" + + "\n" + + "\n" + ); + + Assert.That(() => GrammarValidation.Validate(hostile, dtd), Throws.TypeOf()); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs new file mode 100644 index 000000000..a431ce8b1 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceMorpherFactoryTests.cs @@ -0,0 +1,38 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ConformanceMorpherFactoryTests +{ + [Test] + public void DefaultMorpherIsSequentialAndUntraced() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + + Morpher morpher = ConformanceMorpherFactory.Create(language, fixture.Words.Morpher); + + Assert.Multiple(() => + { + Assert.That(morpher.MaxDegreeOfParallelism, Is.EqualTo(1)); + Assert.That(morpher.TraceManager.IsTracing, Is.False); + }); + } + + [Test] + public void TracingMorpherCanBeCreatedSequentially() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + + Morpher morpher = ConformanceMorpherFactory.CreateTracing(language, fixture.Words.Morpher); + + Assert.Multiple(() => + { + Assert.That(morpher.MaxDegreeOfParallelism, Is.EqualTo(1)); + Assert.That(morpher.TraceManager.IsTracing, Is.True); + }); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs new file mode 100644 index 000000000..431cd67f3 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ConformanceSuite.cs @@ -0,0 +1,93 @@ +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// Finds the conformance suite for the tests that read it: the clone in conformance/ at the commit in +/// conformance.commit, or the directory named by HC_CONFORMANCE_SUITE. +internal static class ConformanceSuite +{ + public const string OverrideVariable = "HC_CONFORMANCE_SUITE"; + private const string SuiteUrl = "https://github.com/sillsdev/fieldworks-parser-conformance.git"; + + private static string? VerifiedClone; + + /// The suite directory. Fails the running test, naming the command that fixes it, when the clone is + /// missing or is not at the pinned commit. A suite named by HC_CONFORMANCE_SUITE is used as it is. + public static string Root() + { + string? overridden = Environment.GetEnvironmentVariable(OverrideVariable); + if (!string.IsNullOrWhiteSpace(overridden)) + { + string directory = Path.GetFullPath(overridden); + if (!File.Exists(Path.Combine(directory, "constructs.txt"))) + Assert.Fail($"{OverrideVariable}={directory} is not the conformance suite: it has no constructs.txt."); + return directory; + } + + if (VerifiedClone is not null) + return VerifiedClone; + + string repositoryRoot = RepositoryRoot(); + string suite = Path.Combine(repositoryRoot, "conformance"); + string pinFile = Path.Combine(repositoryRoot, "conformance.commit"); + string pin = File.Exists(pinFile) ? File.ReadAllText(pinFile).Trim() : ""; + string manual = + $"{Environment.NewLine} git clone --filter=blob:none {SuiteUrl} conformance" + + $"{Environment.NewLine} git -C conformance checkout --detach {pin}"; + + if (!File.Exists(Path.Combine(suite, "constructs.txt"))) + { + Assert.Fail($"The conformance suite is missing: {suite} does not hold it. Run these two commands:{manual}"); + return suite; + } + + ProcessStartInfo startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = suite; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.ArgumentList.Add("rev-parse"); + startInfo.ArgumentList.Add("HEAD"); + string head; + try + { + using Process process = Process.Start(startInfo)!; + head = process.StandardOutput.ReadToEnd().Trim(); + process.WaitForExit(); + if (process.ExitCode != 0) + head = ""; + } + catch (System.ComponentModel.Win32Exception) + { + head = ""; + } + + if (head != pin) + { + Assert.Fail( + $"The conformance suite in {suite} is at '{head}', not the commit in conformance.commit ({pin}). " + + $"Run these two commands:{Environment.NewLine} git -C conformance fetch origin" + + $"{Environment.NewLine} git -C conformance checkout --detach {pin}" + ); + } + + VerifiedClone = suite; + return suite; + } + + /// The directory holding Machine.sln; it does not need the suite. + public static string RepositoryRoot() + { + string? directory = TestContext.CurrentContext.TestDirectory; + while (directory is not null) + { + if (File.Exists(Path.Combine(directory, "Machine.sln"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root (the directory holding Machine.sln)."); + return string.Empty; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs new file mode 100644 index 000000000..66d6c6386 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/DeadRuleAllowlistTests.cs @@ -0,0 +1,156 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class DeadRuleAllowlistTests +{ + private static CoverageReport.DeadRule Dead(string fixtureId, string ruleId) => + new() { FixtureId = fixtureId, RuleId = ruleId }; + + private static string WriteAllowlist(params string[] lines) + { + string path = Path.GetTempFileName(); + File.WriteAllLines(path, lines); + return path; + } + + [Test] + public void AllowedDeadRulesPass() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "r1")] = "decoy" }; + + var (unexpected, stale) = CoverageReport.CheckDeadRules(new[] { Dead("edge-cases/a", "r1") }, allowed); + + Assert.That(unexpected, Is.Empty); + Assert.That(stale, Is.Empty); + } + + [Test] + public void DeadRuleMissingFromAllowlistIsUnexpected() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "r1")] = "decoy" }; + + var (unexpected, _) = CoverageReport.CheckDeadRules( + new[] { Dead("edge-cases/a", "r1"), Dead("edge-cases/a", "r2") }, + allowed + ); + + Assert.That(unexpected.Select(d => d.RuleId), Is.EqualTo(new[] { "r2" })); + } + + [Test] + public void AllowedRuleThatIsNoLongerDeadIsStale() + { + var allowed = new Dictionary<(string, string), string> + { + [("edge-cases/a", "r1")] = "decoy", + [("edge-cases/b", "r1")] = "decoy", + }; + + var (_, stale) = CoverageReport.CheckDeadRules(new[] { Dead("edge-cases/a", "r1") }, allowed); + + Assert.That(stale, Is.EqualTo(new[] { ("edge-cases/b", "r1") })); + } + + [Test] + public void RuleIdsMatchPerFixtureAndOrdinally() + { + var allowed = new Dictionary<(string, string), string> { [("edge-cases/a", "rule")] = "decoy" }; + + var (unexpected, stale) = CoverageReport.CheckDeadRules( + new[] { Dead("edge-cases/b", "rule"), Dead("edge-cases/a", "Rule") }, + allowed + ); + + Assert.That(unexpected, Has.Count.EqualTo(2)); + Assert.That(stale, Has.Count.EqualTo(1)); + } + + [Test] + public void LoaderSkipsCommentsAndBlankLines() + { + string path = WriteAllowlist("# header", "", "edge-cases/a\tr1\tdecoy"); + try + { + Dictionary<(string FixtureId, string RuleId), string> allowed = CoverageReport.LoadDeadRuleAllowlist(path); + + Assert.That(allowed[("edge-cases/a", "r1")], Is.EqualTo("decoy")); + Assert.That(allowed, Has.Count.EqualTo(1)); + } + finally + { + File.Delete(path); + } + } + + [TestCase("edge-cases/a\tr1")] + [TestCase("edge-cases/a\tr1\t ")] + [TestCase("edge-cases/a\tr1\tdecoy\textra")] + public void LoaderRejectsRowsWithoutAReason(string row) + { + string path = WriteAllowlist(row); + try + { + Assert.Throws(() => CoverageReport.LoadDeadRuleAllowlist(path)); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void LoaderRejectsDuplicateEntries() + { + string path = WriteAllowlist("edge-cases/a\tr1\tdecoy", "edge-cases/a\tr1\tagain"); + try + { + Assert.Throws(() => CoverageReport.LoadDeadRuleAllowlist(path)); + } + finally + { + File.Delete(path); + } + } + + private static WordsYaml WordsBlockedBy(params string[] labels) + { + var words = new WordsYaml { Language = "L" }; + var word = new WordEntry { Word = "w", ExpectFail = true }; + word.BlockedBy.AddRange(labels); + words.Words.Add(word); + return words; + } + + [Test] + public void BlockedByLabelThatIsAGrammarRuleIdPasses() + { + var unknown = CoverageReport.FindUnknownBlockedBy("edge-cases/a", WordsBlockedBy("r1"), new[] { "r1", "r2" }); + + Assert.That(unknown, Is.Empty); + } + + [Test] + public void BlockedByLabelThatIsNotAGrammarRuleIdFails() + { + var unknown = CoverageReport.FindUnknownBlockedBy( + "edge-cases/a", + WordsBlockedBy("r1", "bogus"), + new[] { "r1", "r2" } + ); + + Assert.That(unknown.Select(u => u.Label), Is.EqualTo(new[] { "bogus" })); + Assert.That(unknown[0].FixtureId, Is.EqualTo("edge-cases/a")); + Assert.That(unknown[0].Word, Is.EqualTo("w")); + } + + [Test] + public void BlockedByLabelsAreComparedOrdinally() + { + var unknown = CoverageReport.FindUnknownBlockedBy("edge-cases/a", WordsBlockedBy("R1"), new[] { "r1" }); + + Assert.That(unknown, Has.Count.EqualTo(1)); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs new file mode 100644 index 000000000..968353535 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksProducibleShapeTests.cs @@ -0,0 +1,895 @@ +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A fixture flagged fieldworks_producible: true must not use a grammar shape HCLoader never +/// writes. The flag is authored, so these checks can only prove it wrong. Most of the checks come from the +/// census table conformance/fieldworks-writable-shapes.tsv, which says per rule kind which DTD members +/// HCLoader writes; reads it. The checks written here are the ones a +/// table row cannot say: a linear stratum that holds rules or templates, and a partial stem entry +/// with a part of speech. Each class of violation has one test that must fail on a synthetic grammar. +/// +[TestFixture] +public sealed class FieldWorksProducibleShapeTests +{ + private static readonly Lazy> LoadedTable = new(() => + FieldWorksWritableShapes.LoadTable(FieldWorksWritableShapes.TablePath(ConformanceSuite.Root())) + ); + + private static Dictionary<(string Kind, string Member), string> Table => LoadedTable.Value; + + // The 16 stress fixtures sit in the suite under stress/ and obey the same FieldWorks-producible rule. + private static List DiscoverProducible() + { + string root = ConformanceSuite.Root(); + var fixtures = Fixture.DiscoverAll(root); + fixtures.AddRange(Fixture.DiscoverAll(Path.Combine(root, "stress"))); + return fixtures.Where(fixture => fixture.Words.FieldworksProducible == true).ToList(); + } + + [Test] + public void FlaggedFixturesUseNoShapeHcLoaderCannotWrite() + { + List fixtures = DiscoverProducible(); + Assert.That(fixtures, Has.Count.GreaterThanOrEqualTo(25)); + + string[] offences = fixtures + .SelectMany(fixture => FindUnproducibleShapes(fixture.GrammarPath).Select(o => $"{fixture.Id}: {o}")) + .OrderBy(line => line, StringComparer.Ordinal) + .ToArray(); + + Assert.That( + offences, + Is.Empty, + "these fixtures are flagged fieldworks_producible but use a shape HCLoader never writes; " + + "rewrite the grammar and re-run the oracle, or delete the fixture:\n " + + string.Join("\n ", offences) + ); + } + + [Test] + public void TheCensusTableNamesOnlyMembersTheDtdDeclares() + { + string dtd = File.ReadAllText(Path.Combine(ConformanceSuite.Root(), "HermitCrabInput.dtd")); + var elements = new HashSet( + Regex.Matches(dtd, @" m.Groups[1].Value), + StringComparer.Ordinal + ); + var attributes = new HashSet(StringComparer.Ordinal); + foreach (Match list in Regex.Matches(dtd, @"", RegexOptions.Singleline)) + { + foreach (Match attribute in Regex.Matches(list.Groups[1].Value, @"^\s*(\w+)\s", RegexOptions.Multiline)) + attributes.Add(attribute.Groups[1].Value); + } + + string[] unknown = Table + .Keys.Select(key => key.Member) + .Distinct(StringComparer.Ordinal) + .Where(member => member != "*") + .Where(member => + member.StartsWith('@') + ? !attributes.Contains(member.TrimStart('@').Split('=')[0]) + : !elements.Contains(member) + ) + .OrderBy(member => member, StringComparer.Ordinal) + .ToArray(); + + Assert.That(unknown, Is.Empty, "the census table names members the DTD does not declare"); + } + + [Test] + public void TheCensusTableGivesALoaderLineToEveryRowThatSaysHcLoaderWritesIt() + { + string[] rows = File.ReadAllLines(FieldWorksWritableShapes.TablePath(ConformanceSuite.Root())) + .Skip(1) + .ToArray(); + string[] missing = rows.Select(row => row.Split('\t')) + .Where(cells => cells[2] is "writes" or "never" or "omitted" && cells[3].Length == 0) + .Select(cells => $"{cells[0]} {cells[1]}") + .OrderBy(row => row, StringComparer.Ordinal) + .ToArray(); + + Assert.That(missing, Is.Empty, "census rows without a HCLoader.cs line"); + } + + [Test] + public void TheCheckFindsAMemberTheCensusTableDoesNotList() + { + string[] found = Scan( + """ + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "derivational-rule mrNew uses @newAttribute, which the census table does not list" }) + ); + } + + [Test] + public void TheCheckFindsARealizationalRule() + { + string[] found = Scan( + """ + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "realizational-rule rrOne is never written by HCLoader (HCLoader.cs 1027)" }) + ); + } + + [Test] + public void TheCheckFindsASlotRuleWithAnOutputPartOfSpeech() + { + string[] found = Scan( + """ + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "slot-rule mrSlot uses @outputPartOfSpeech, which HCLoader never writes" }) + ); + } + + [Test] + public void TheCheckFindsOutputFeaturesOutputMprAndARequiredStemNameOnASlotRule() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "slot-rule mrSlot uses @requiredStemName, which HCLoader never writes", + "slot-rule mrSlot uses OutputHeadFeatures, which HCLoader never writes", + "slot-rule/output sub1 uses @MPRFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsHeadFeatureRequirementsOnACompoundingRule() + { + string[] found = Scan( + """ + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crHead uses HeadRequiredHeadFeatures, which HCLoader never writes", + "compounding-rule crHead uses NonHeadRequiredHeadFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsAnEmptyStemNameRegion() + { + string[] found = Scan( + """ + + """, + """ + + e + + f + + + g + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "stem name snEmpty has an empty region, which HCLoader skips (HCLoader.cs 223)", + "stem name snNoFeatures has an empty region, which HCLoader skips (HCLoader.cs 223)", + } + ) + ); + } + + [Test] + public void TheCheckFindsARequiredEnvironmentOnASubruleShapeHcLoaderGivesNone() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule infix has a RequiredEnvironments on an output of CopyFromInput, InsertSegments, " + + "CopyFromInput; HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix", + "subrule process has a RequiredEnvironments on an output of CopyFromInput, ModifyFromInput; " + + "HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix", + } + ) + ); + } + + [Test] + public void TheCheckFindsRequiredHeadFeaturesOnASubruleShapeHcLoaderGivesNone() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule circumfix is a circumfix with RequiredHeadFeatures; HCLoader writes it only on a lone affix form", + "subrule process is an affix process with RequiredHeadFeatures; HCLoader writes it only on a lone affix form", + } + ) + ); + } + + [Test] + public void TheCheckFindsRequiredHeadFeaturesOnACliticSubrule() + { + string[] found = Scan( + """ + + Clitics + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo(new[] { "clitic-rule/subrule enclitic uses RequiredHeadFeatures, which HCLoader never writes" }) + ); + } + + [Test] + public void TheCheckFindsACapOnAnExocompound() + { + string[] found = Scan( + """ + + + exo + exo + endo + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding rule exoLeft is half of an exocompound, which HCLoader never gives a cap; " + + "it has multipleApplication=\"3\"", + } + ) + ); + } + + [Test] + public void TheCheckFindsMprFeaturesHcLoaderCannotPlace() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + """, + """ + + + + + + inflClasses + + exceptionFeatures + + custom + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "MPR feature group custom is not one of the three groups HCLoader builds", + "MPR feature group inflClasses has matchType=\"all\"; HCLoader builds \"any\"", + "MPR feature group inflClasses has outputType=\"append\", which HCLoader never sets", + "MPR feature mprLoose is in no group; HCLoader puts every MPR feature in one of three groups", + "derivational rule mrDeriv has different MPR lists on its subrules; " + + "HCLoader gives every allomorph the same list (HCLoader.cs 1015-1018)", + "derivational-rule/input subB uses @excludedMPRFeatures, which HCLoader never writes", + "rule mrDeriv excludes MPR feature mprExc; HCLoader excludes only the inflection-type " + + "features of a slot (HCLoader.cs 1766)", + "rule mrSlot excludes MPR feature mprExc; HCLoader excludes only the inflection-type " + + "features of a slot (HCLoader.cs 1766)", + } + ) + ); + } + + [Test] + public void TheCheckAcceptsAnInflectionTypeFeatureExcludedByASlotRule() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """, + """ + + + + lexEntryInflTypes + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckAcceptsAttributesHcLoaderOmitsOrNeverReads() + { + string[] found = Scan( + """ + + + + + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckFindsALinearStratumThatHoldsAMorphologicalRuleOrATemplate() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "stratum #1 is linear (the DTD default) but holds morphological rules mrA", + "stratum #2 is linear but holds morphological rules mrB", + "stratum #3 is linear but holds 1 affix template(s)", + } + ) + ); + } + + [Test] + public void TheCheckAllowsLinearStrataThatHoldNoActiveRuleOrTemplate() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + """ + ); + + Assert.That(found, Is.Empty); + } + + [Test] + public void TheCheckFindsPartialShapesHcLoaderCannotWrite() + { + string[] found = Scan( + """ + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "partial stem entry eBad has a partOfSpeech", + "slot-rule mrSlot uses @partial=true, which HCLoader never writes", + "unslotted-rule mrLoose uses @outputPartOfSpeech, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsFootFeatures() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + """, + "" + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crFoot uses HeadRequiredFootFeatures, which HCLoader never writes", + "derivational-rule mrFoot uses OutputFootFeatures, which HCLoader never writes", + "derivational-rule mrFoot uses RequiredFootFeatures, which HCLoader never writes", + "lexical-entry eFoot uses AssignedFootFeatures, which HCLoader never writes", + "the language defines FootFeatures, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsExcludedEnvironments() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "derivational-rule/subrule subExcl uses ExcludedEnvironments, which HCLoader never writes", + "root-allomorph aExcl uses ExcludedEnvironments, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsARuleThatIsNotBlockable() + { + string[] found = Scan( + """ + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crNo uses @blockable=false, which HCLoader never writes", + "derivational-rule mrNo uses @blockable=false, which HCLoader never writes", + } + ) + ); + } + + [Test] + public void TheCheckFindsAWrongSideEnvironmentOnALonePrefixOrSuffix() + { + string[] found = Scan( + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "subrule prefixRightBad is a lone prefix with a right environment; HCLoader writes only a left one", + "subrule suffixBothBad is a lone suffix with a left environment; HCLoader writes only a right one", + "subrule suffixLeftBad is a lone suffix with a left environment; HCLoader writes only a right one", + } + ) + ); + } + + [Test] + public void TheCheckFindsAMaximumApplicationCountHcLoaderCannotSet() + { + string[] found = Scan( + """ + + + + + + + + + + """ + ); + + Assert.That( + found, + Is.EqualTo( + new[] + { + "compounding-rule crZero uses @multipleApplication=0, which HCLoader never writes", + "derivational-rule mrTwice uses @multipleApplication=2, which HCLoader never writes", + } + ) + ); + } + + private static string[] Scan(string strata, string languageExtras = "") + { + string path = Path.Combine(Path.GetTempPath(), $"hc-shape-{Guid.NewGuid():N}.xml"); + File.WriteAllText( + path, + $"{languageExtras}{strata}" + ); + try + { + return FindUnproducibleShapes(path); + } + finally + { + File.Delete(path); + } + } + + private static readonly string[] RuleElements = ["MorphologicalRule", "RealizationalRule", "CompoundingRule"]; + + private static string[] FindUnproducibleShapes(string grammarPath) + { + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + using XmlReader reader = XmlReader.Create(grammarPath, settings); + XDocument doc = XDocument.Load(reader); + + List offences = FieldWorksWritableShapes.Check(doc, Table); + + foreach (XElement entry in doc.Descendants("LexicalEntry")) + { + if ((string?)entry.Attribute("partial") == "true" && entry.Attribute("partOfSpeech") is not null) + offences.Add($"partial stem entry {(string?)entry.Attribute("id")} has a partOfSpeech"); + } + + // HCLoader builds every stratum that holds rules or templates as Unordered, and the DTD default is + // linear. The engine reads the order for the listed rules and when it applies templates. + int number = 0; + foreach (XElement stratum in doc.Descendants("Stratum")) + { + number++; + string? order = (string?)stratum.Attribute("morphologicalRuleOrder"); + if (order == "unordered") + continue; + + var active = new HashSet(StringComparer.Ordinal); + foreach ( + XElement rule in stratum + .Elements("MorphologicalRuleDefinitions") + .Elements() + .Where(e => RuleElements.Contains(e.Name.LocalName)) + ) + { + string? id = (string?)rule.Attribute("id"); + if (id is not null && (string?)rule.Attribute("isActive") != "no") + active.Add(id); + } + + string how = order is null ? "linear (the DTD default)" : "linear"; + string[] held = Split((string?)stratum.Attribute("morphologicalRules")).Where(active.Contains).ToArray(); + if (held.Length > 0) + offences.Add($"stratum #{number} is {how} but holds morphological rules {string.Join(" ", held)}"); + + int templates = stratum + .Elements("AffixTemplates") + .Elements("AffixTemplate") + .Count(template => (string?)template.Attribute("isActive") != "no"); + if (templates > 0) + offences.Add($"stratum #{number} is {how} but holds {templates} affix template(s)"); + } + + return offences.OrderBy(line => line, StringComparer.Ordinal).ToArray(); + } + + private static string[] Split(string? ids) => + string.IsNullOrWhiteSpace(ids) ? [] : ids.Split(' ', StringSplitOptions.RemoveEmptyEntries); +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs new file mode 100644 index 000000000..f2dff0dbf --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/FieldWorksWritableShapes.cs @@ -0,0 +1,432 @@ +using System.Xml.Linq; +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// The check behind conformance/fieldworks-writable-shapes.tsv, the census of what HCLoader can write. +/// Every element and attribute a flagged grammar puts on a rule, template, stratum, stem name, entry, allomorph +/// or phonological rule is looked up there by the kind it sits on. A member the table calls never, or +/// does not list, is an offence. The conditions a table row cannot say (an empty stem-name region, where an +/// environment may sit, which MPR feature group a feature belongs to, a cap on an exocompound) are checked here. +/// +internal static class FieldWorksWritableShapes +{ + private static readonly string[] EnumAttributes = + [ + "isActive", + "partial", + "blockable", + "final", + "optional", + "morphologicalRuleOrder", + "cyclicity", + "phonologicalRuleOrder", + "multipleApplication", + "multipleApplicationOrder", + "isBound", + "redupMorphType", + "type", + "adjacency", + ]; + + // LoadLanguage (HCLoader.cs 181-205) builds exactly these three groups, all with the default output. + private static readonly Dictionary LoaderGroups = new(StringComparer.Ordinal) + { + ["inflClasses"] = "any", + ["exceptionFeatures"] = "all", + ["lexEntryInflTypes"] = "all", + }; + + private static readonly string[] Verdicts = ["writes", "metadata", "never", "omitted", "inert", "shape"]; + + public static string TablePath(string suiteRoot) => Path.Combine(suiteRoot, "fieldworks-writable-shapes.tsv"); + + public static Dictionary<(string Kind, string Member), string> LoadTable(string path) + { + var table = new Dictionary<(string, string), string>(); + string[] lines = File.ReadAllLines(path); + Assert.That(lines[0], Is.EqualTo("kind\tmember\tverdict\tloader\tnote")); + foreach (string line in lines.Skip(1).Where(l => l.Length > 0)) + { + string[] cells = line.Split('\t'); + Assert.That(cells, Has.Length.EqualTo(5), line); + Assert.That(Verdicts, Does.Contain(cells[2]), line); + Assert.That(table.TryAdd((cells[0], cells[1]), cells[2]), Is.True, $"duplicate row: {line}"); + } + return table; + } + + public static IEnumerable<(string Kind, string Member, string Verdict)> Rows( + Dictionary<(string Kind, string Member), string> table + ) => table.Select(pair => (pair.Key.Kind, pair.Key.Member, pair.Value)); + + public static List Check(XDocument doc, Dictionary<(string Kind, string Member), string> table) + { + var offences = new List(); + var slotRuleIds = new HashSet(StringComparer.Ordinal); + foreach (XElement slot in doc.Descendants("Slot")) + slotRuleIds.UnionWith(Split((string?)slot.Attribute("morphologicalRules"))); + + var cliticRuleIds = new HashSet(StringComparer.Ordinal); + foreach (XElement stratum in doc.Descendants("Stratum").Where(s => (string?)s.Element("Name") == "Clitics")) + cliticRuleIds.UnionWith(Split((string?)stratum.Attribute("morphologicalRules"))); + + foreach (XElement element in doc.Descendants()) + { + string? kind = KindOf(element, slotRuleIds, cliticRuleIds); + if (kind is null) + continue; + if (kind == "realizational-rule") + { + offences.Add($"{kind} {Label(element)} is never written by HCLoader (HCLoader.cs 1027)"); + continue; + } + + foreach ((string specific, string generic) in Members(element)) + { + if (!table.TryGetValue((kind, specific), out string? verdict)) + table.TryGetValue((kind, generic), out verdict); + if (verdict is null) + offences.Add($"{kind} {Label(element)} uses {specific}, which the census table does not list"); + else if (verdict == "never") + offences.Add($"{kind} {Label(element)} uses {specific}, which HCLoader never writes"); + } + } + + if (doc.Descendants("FootFeatures").Any()) + offences.Add("the language defines FootFeatures, which HCLoader never writes"); + + CheckStemNameRegions(doc, offences); + CheckEnvironmentShapes(doc, offences); + CheckSubruleHeadFeatureShapes(doc, offences); + CheckExocompoundCaps(doc, offences); + CheckMprFeatureGroups(doc, slotRuleIds, offences); + return offences; + } + + private static IEnumerable<(string Specific, string Generic)> Members(XElement element) + { + foreach (XAttribute attribute in element.Attributes().Where(a => !a.IsNamespaceDeclaration)) + { + string name = attribute.Name.LocalName; + yield return EnumAttributes.Contains(name) + ? ($"@{name}={attribute.Value}", $"@{name}") + : ($"@{name}", $"@{name}"); + } + foreach (XElement child in element.Elements()) + yield return (child.Name.LocalName, child.Name.LocalName); + } + + private static string Label(XElement element) + { + for (XElement? current = element; current is not null; current = current.Parent) + { + string? label = (string?)current.Attribute("id") ?? (string?)current.Element("Name"); + if (label is not null) + return label; + } + return "?"; + } + + private static string? KindOf(XElement element, HashSet slotRuleIds, HashSet cliticRuleIds) + { + switch (element.Name.LocalName) + { + case "MorphologicalRule": + return AffixRuleKind(element, slotRuleIds, cliticRuleIds); + case "CompoundingRule": + return "compounding-rule"; + case "RealizationalRule": + return "realizational-rule"; + case "MorphologicalSubrule": + return AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/subrule"; + case "MorphologicalInput": + return AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/input"; + case "MorphologicalOutput": + return element.Parent?.Name.LocalName == "CompoundingSubrule" + ? "compounding-rule/output" + : AffixRuleKindOf(element, slotRuleIds, cliticRuleIds) + "/output"; + case "CompoundingSubrule": + return "compounding-rule/subrule"; + case "HeadMorphologicalInput": + return "compounding-rule/head-input"; + case "NonHeadMorphologicalInput": + return "compounding-rule/nonhead-input"; + case "AffixTemplate": + return "template"; + case "Slot": + return "slot"; + case "Stratum": + return "stratum"; + case "StemName": + return "stem-name"; + case "Region": + return "stem-region"; + case "LexicalEntry": + return "lexical-entry"; + case "Allomorph": + return "root-allomorph"; + case "PhonologicalRule": + return "phonological-rule"; + case "PhonologicalSubrule": + return "phonological-subrule"; + case "MetathesisRule": + return "metathesis-rule"; + case "MorphemeCoOccurrenceRule": + case "AllomorphCoOccurrenceRule": + return "cooccurrence-rule"; + default: + return null; + } + } + + private static string AffixRuleKindOf(XElement inner, HashSet slotRuleIds, HashSet cliticRuleIds) + { + XElement? rule = inner.Ancestors("MorphologicalRule").FirstOrDefault(); + return rule is null ? "unknown-rule" : AffixRuleKind(rule, slotRuleIds, cliticRuleIds); + } + + // A rule in a slot is inflectional, a rule of the Clitics stratum is a clitic rule, a partial rule outside + // a slot is unslotted, and the rest are derivational (HCLoader.cs 926-1093). + private static string AffixRuleKind(XElement rule, HashSet slotRuleIds, HashSet cliticRuleIds) + { + string id = (string?)rule.Attribute("id") ?? ""; + if (slotRuleIds.Contains(id)) + return "slot-rule"; + if (cliticRuleIds.Contains(id)) + return "clitic-rule"; + return (string?)rule.Attribute("partial") == "true" ? "unslotted-rule" : "derivational-rule"; + } + + // HCLoader keeps a stem name only when it has a non-empty region (HCLoader.cs 223), so an empty region is a + // grammar FieldWorks cannot write: it would leave no stem name at all. + private static void CheckStemNameRegions(XDocument doc, List offences) + { + foreach (XElement region in doc.Descendants("Region")) + { + XElement? head = region.Element("AssignedHeadFeatures"); + if (head is null || !head.Elements("FeatureValue").Any()) + offences.Add($"stem name {Label(region)} has an empty region, which HCLoader skips (HCLoader.cs 223)"); + } + } + + // HCLoader writes an environment on a lone prefix (left), a lone suffix (right) and a circumfix (both). + // An infix, a reduplication and an affix process add none (HCLoader.cs 1371, 1576-1654). + private static void CheckEnvironmentShapes(XDocument doc, List offences) + { + foreach (XElement subrule in doc.Descendants("MorphologicalSubrule")) + { + XElement? required = subrule.Element("RequiredEnvironments"); + if (required is null) + continue; + + string[] output = (subrule.Element("MorphologicalOutput")?.Elements() ?? []) + .Select(e => e.Name.LocalName) + .ToArray(); + bool left = required.Descendants("LeftEnvironment").Any(); + bool right = required.Descendants("RightEnvironment").Any(); + string id = (string?)subrule.Attribute("id") ?? "?"; + if (output.SequenceEqual(["InsertSegments", "CopyFromInput"])) + { + if (right) + { + offences.Add( + $"subrule {id} is a lone prefix with a right environment; HCLoader writes only a left one" + ); + } + } + else if (output.SequenceEqual(["CopyFromInput", "InsertSegments"])) + { + if (left) + { + offences.Add( + $"subrule {id} is a lone suffix with a left environment; HCLoader writes only a right one" + ); + } + } + else if (!output.SequenceEqual(["InsertSegments", "CopyFromInput", "InsertSegments"])) + { + offences.Add( + $"subrule {id} has a RequiredEnvironments on an output of {string.Join(", ", output)}; " + + "HCLoader writes an environment only for a lone prefix, a lone suffix or a circumfix" + ); + } + } + } + + // HCLoader sets a subrule's head features only on a lone affix form, not a circumfix (1100-1127) or an affix + // process (ModifyFromInput, 1143, 1458-1466). + private static void CheckSubruleHeadFeatureShapes(XDocument doc, List offences) + { + foreach (XElement subrule in doc.Descendants("MorphologicalSubrule")) + { + if (subrule.Element("RequiredHeadFeatures") is null) + continue; + + string[] output = (subrule.Element("MorphologicalOutput")?.Elements() ?? []) + .Select(e => e.Name.LocalName) + .ToArray(); + string id = (string?)subrule.Attribute("id") ?? "?"; + if (output.SequenceEqual(["InsertSegments", "CopyFromInput", "InsertSegments"])) + { + offences.Add( + $"subrule {id} is a circumfix with RequiredHeadFeatures; HCLoader writes it only on a lone affix form" + ); + } + else if (output.Contains("ModifyFromInput")) + { + offences.Add( + $"subrule {id} is an affix process with RequiredHeadFeatures; HCLoader writes it only on a lone affix form" + ); + } + } + } + + // An exocompound becomes two mirror rules with the same name and no cap (HCLoader.cs 1971-2050). + // Only an endocompound reads the cap, from the parser parameters (1943-1945). + private static void CheckExocompoundCaps(XDocument doc, List offences) + { + XElement[] compounds = doc.Descendants("CompoundingRule").ToArray(); + foreach (XElement rule in compounds) + { + string? cap = (string?)rule.Attribute("multipleApplication"); + if (cap is null or "1") + continue; + if (compounds.Any(other => !ReferenceEquals(other, rule) && IsMirror(rule, other))) + { + offences.Add( + $"compounding rule {Label(rule)} is half of an exocompound, which HCLoader never gives a cap; " + + $"it has multipleApplication=\"{cap}\"" + ); + } + } + } + + private static bool IsMirror(XElement a, XElement b) => + (string?)a.Element("Name") is not null + && (string?)a.Element("Name") == (string?)b.Element("Name") + && ((string?)a.Attribute("headPartsOfSpeech") ?? "") == ((string?)b.Attribute("nonHeadPartsOfSpeech") ?? "") + && ((string?)a.Attribute("nonHeadPartsOfSpeech") ?? "") == ((string?)b.Attribute("headPartsOfSpeech") ?? "") + && (string?)a.Attribute("outputPartOfSpeech") == (string?)b.Attribute("outputPartOfSpeech"); + + // HCLoader puts each MPR feature in inflClasses, exceptionFeatures or lexEntryInflTypes (181-205). + // A slot rule excludes only inflection types (1766); an entry has one inflection class (745-751). + private static void CheckMprFeatureGroups(XDocument doc, HashSet slotRuleIds, List offences) + { + XElement? features = doc.Descendants("MorphologicalPhonologicalRuleFeatures").FirstOrDefault(); + if (features is null) + return; + + var groupOf = new Dictionary(StringComparer.Ordinal); + foreach (XElement group in features.Elements("MorphologicalPhonologicalRuleFeatureGroup")) + { + string name = (string?)group.Element("Name") ?? "?"; + string matchType = (string?)group.Attribute("matchType") ?? "any"; + string outputType = (string?)group.Attribute("outputType") ?? "overwrite"; + if (!LoaderGroups.TryGetValue(name, out string? expected)) + offences.Add($"MPR feature group {name} is not one of the three groups HCLoader builds"); + else if (expected != matchType) + offences.Add($"MPR feature group {name} has matchType=\"{matchType}\"; HCLoader builds \"{expected}\""); + if (outputType != "overwrite") + offences.Add($"MPR feature group {name} has outputType=\"{outputType}\", which HCLoader never sets"); + foreach (string feature in Split((string?)group.Attribute("features"))) + groupOf[feature] = name; + } + + foreach (XElement feature in features.Elements("MorphologicalPhonologicalRuleFeature")) + { + string id = (string?)feature.Attribute("id") ?? "?"; + if (!groupOf.ContainsKey(id)) + { + offences.Add( + $"MPR feature {id} is in no group; HCLoader puts every MPR feature in one of three groups" + ); + } + } + + string GroupOf(string feature) => groupOf.TryGetValue(feature, out string? g) ? g : ""; + + foreach (XElement rule in doc.Descendants("MorphologicalRule")) + { + string id = (string?)rule.Attribute("id") ?? "?"; + bool inSlot = slotRuleIds.Contains(id); + var requiredLists = new HashSet(StringComparer.Ordinal); + var outputLists = new HashSet(StringComparer.Ordinal); + foreach (XElement subrule in rule.Descendants("MorphologicalSubrule")) + { + string[] required = Split( + (string?)subrule.Element("MorphologicalInput")?.Attribute("requiredMPRFeatures") + ); + string[] excluded = Split( + (string?)subrule.Element("MorphologicalInput")?.Attribute("excludedMPRFeatures") + ); + string[] output = Split((string?)subrule.Element("MorphologicalOutput")?.Attribute("MPRFeatures")); + requiredLists.Add(string.Join(" ", required.OrderBy(f => f, StringComparer.Ordinal))); + outputLists.Add(string.Join(" ", output.OrderBy(f => f, StringComparer.Ordinal))); + + foreach (string feature in excluded.Where(f => !inSlot || GroupOf(f) != "lexEntryInflTypes")) + { + offences.Add( + $"rule {id} excludes MPR feature {feature}; HCLoader excludes only the " + + "inflection-type features of a slot (HCLoader.cs 1766)" + ); + } + foreach (string feature in required.Concat(output).Where(f => GroupOf(f) == "lexEntryInflTypes")) + { + offences.Add( + $"rule {id} requires or writes inflection-type MPR feature {feature}; HCLoader never does" + ); + } + if (output.Count(f => GroupOf(f) == "inflClasses") > 1) + offences.Add($"rule {id} writes more than one inflection class; a derivational affix writes one"); + } + + bool derivational = !inSlot && (string?)rule.Attribute("partial") != "true"; + if (derivational && (requiredLists.Count > 1 || outputLists.Count > 1)) + { + offences.Add( + $"derivational rule {id} has different MPR lists on its subrules; " + + "HCLoader gives every allomorph the same list (HCLoader.cs 1015-1018)" + ); + } + } + + foreach (XElement entry in doc.Descendants("LexicalEntry")) + { + if (Split((string?)entry.Attribute("ruleFeatures")).Count(f => GroupOf(f) == "inflClasses") > 1) + offences.Add($"entry {Label(entry)} has more than one inflection class"); + } + + foreach (XElement rule in doc.Descendants("CompoundingRule")) + { + foreach ( + XAttribute attribute in rule.Attributes() + .Where(a => a.Name.LocalName.EndsWith("ProdRestrictionsMprFeatures", StringComparison.Ordinal)) + ) + { + foreach (string feature in Split(attribute.Value).Where(f => GroupOf(f) != "exceptionFeatures")) + { + offences.Add( + $"compounding rule {Label(rule)} lists {feature} as a production restriction, " + + "which is not an exceptionFeatures feature" + ); + } + } + foreach (XElement output in rule.Descendants("MorphologicalOutput")) + { + string[] written = Split((string?)output.Attribute("MPRFeatures")); + if (written.Length > 1 || written.Any(f => GroupOf(f) != "inflClasses")) + { + offences.Add( + $"compounding rule {Label(rule)} writes {string.Join(" ", written)}; " + + "HCLoader writes one inflection class (HCLoader.cs 1950)" + ); + } + } + } + } + + private static string[] Split(string? ids) => + string.IsNullOrWhiteSpace(ids) ? [] : ids.Split(' ', StringSplitOptions.RemoveEmptyEntries); +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs new file mode 100644 index 000000000..4a9547cae --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/GuesserSignatureTests.cs @@ -0,0 +1,112 @@ +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class GuesserSignatureTests +{ + private const string GrammarXml = """ + + + + + GuesserSignatureProbe + + n + + + Main + + f + o + s + + + + Any + + + + Main + + + plural + + + + s + + + PL + + + + + + [Any]* + + + + + + + """; + + private static string WriteTempGrammar() + { + string path = Path.Combine(Path.GetTempPath(), "hc-guesser-signature-" + Guid.NewGuid().ToString("N") + ".xml"); + File.WriteAllText(path, GrammarXml); + return path; + } + + // Named for the overload it calls, not the production path: no Tool-project reference means + // it cannot call ParseOneWord. Proves the fact that path relies on: the 2-argument overload + // never guesses. + [Test] + public void TwoArgumentParseWord_NeverGuesses_ZeroResultsForAnUnknownRoot() + { + string path = WriteTempGrammar(); + try + { + Language language = XmlLanguageLoader.Load(path); + var morpher = new Morpher(new TraceManager(), language); + + Word[] results = morpher.ParseWord("foos", out _).ToArray(); + + Assert.That(results, Is.Empty); + Assert.That(SignatureFormat.BuildSignature(results), Is.EqualTo("-")); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void GuessRootTrue_RendersGuessedStemAsItsOwnLiteralSurfaceText() + { + string path = WriteTempGrammar(); + try + { + Language language = XmlLanguageLoader.Load(path); + var morpher = new Morpher(new TraceManager(), language); + + Word[] results = morpher.ParseWord("foos", out _, guessRoot: true).ToArray(); + + Assert.That(results, Is.Not.Empty); + string signature = SignatureFormat.BuildSignature(results); + + Assert.That(signature, Is.EqualTo("foo+PL|foos;foos|foos")); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs new file mode 100644 index 000000000..96d56da1b --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MaterializedRunnerCrashClassificationTests.cs @@ -0,0 +1,136 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A harness-level problem is never a crash match, expectCrash or not. A misconfigured adapter +/// (wrong path, an argument it rejects, ...) exits nonzero on every fixture it is asked to run, not +/// just the one pinning a crash -- these tests use a stub (no real subprocess) +/// to prove tells the two apart. +/// +[TestFixture] +public sealed class MaterializedRunnerCrashClassificationTests +{ + private sealed class StubEngine(params string[] crashingFixtureIds) : IEngine + { + private readonly HashSet _crashingFixtureIds = new(crashingFixtureIds, StringComparer.Ordinal); + + public string Name => "stub"; + public IReadOnlySet Capabilities { get; } = new HashSet(); + + public List Run(MaterializedFixture fixture) + { + if (_crashingFixtureIds.Contains(fixture.Id)) + throw new EngineCrashException($"stub crash for '{fixture.Id}'"); + return new List(); + } + } + + private static MaterializedFixture CreateFixture( + string root, + string id, + bool expectCrash, + bool requiresPhonology = false, + bool pathological = false + ) + { + string directory = Path.Combine(root, id); + Directory.CreateDirectory(directory); + // A PhonologicalRule is what RequiresDerivation.Derive reads as "phonology"; without one the + // manifest's empty Requires matches the grammar. + File.WriteAllText( + Path.Combine(directory, "grammar.xml"), + requiresPhonology ? "" : "" + ); + File.WriteAllText(Path.Combine(directory, "words.txt"), "w\n"); + File.WriteAllText(Path.Combine(directory, "expected.tsv"), ""); + var manifest = new FixtureManifest { Id = id, ExpectCrash = expectCrash }; + if (requiresPhonology) + manifest.Requires.Add("phonology"); + if (pathological) + manifest.Category = "pathological"; + return new MaterializedFixture(directory, manifest); + } + + private static MaterializedRunReport Run(string root, IEngine engine) + { + var fixtures = new List + { + CreateFixture(root, "crash-fixture", expectCrash: true), + CreateFixture(root, "control-fixture", expectCrash: false), + }; + return MaterializedRunner.Run(fixtures, engine, includePathological: false); + } + + [Test] + public void ExpectCrashPassesWhenTheSameEngineStillCompletesAnOrdinaryFixture() + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Genuine crash: the stub only fails on the fixture that pins one. + MaterializedRunReport report = Run(root, new StubEngine("crash-fixture")); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Passed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public void ExpectCrashDoesNotPassWhenTheEngineFailsEveryFixtureNotJustTheCrashOne() + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Misconfigured adapter: rejects its own arguments and exits nonzero regardless of which + // fixture it was asked to run. Must not be credited as reproducing the pinned crash. + MaterializedRunReport report = Run(root, new StubEngine("crash-fixture", "control-fixture")); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Failed), result.Reason); + Assert.That(result.Reason, Does.Contain("control-fixture")); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [TestCase(true, false, TestName = "ExpectCrashSkipsAControlTheEngineLacksACapabilityFor")] + [TestCase(false, true, TestName = "ExpectCrashSkipsAControlExcludedAsPathological")] + public void ExpectCrashUsesOnlyAControlTheRunWouldExecute(bool requiresPhonology, bool pathological) + { + string root = Path.Combine(Path.GetTempPath(), $"hc-crash-classification-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // The stub declares no capabilities and fails the ineligible fixture, as an engine that + // cannot run it would. Only the eligible control says whether the adapter works. + var fixtures = new List + { + CreateFixture(root, "crash-fixture", expectCrash: true), + CreateFixture(root, "ineligible-control", expectCrash: false, requiresPhonology, pathological), + CreateFixture(root, "control-fixture", expectCrash: false), + }; + MaterializedRunReport report = MaterializedRunner.Run( + fixtures, + new StubEngine("crash-fixture", "ineligible-control"), + includePathological: false + ); + + FixtureResult result = report.Results.Single(r => r.FixtureId == "crash-fixture"); + Assert.That(result.Outcome, Is.EqualTo(FixtureOutcome.Passed), result.Reason); + } + finally + { + Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs index d9149d058..4028a425d 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs @@ -11,6 +11,17 @@ namespace SIL.Machine.Morphology.HermitCrab; [TestFixture] public class MorpherTests : HermitCrabTestBase { + [Test] + public void Constructor_DefaultsMatchDocumentedProtocolDefaults() + { + var morpher = new Morpher(TraceManager, Language); + + Assert.That(morpher.MaxStemCount, Is.EqualTo(2)); + Assert.That(morpher.DeletionReapplications, Is.EqualTo(0)); + Assert.That(morpher.MaxAlternatives, Is.EqualTo(0)); + Assert.That(morpher.MergeEquivalentAnalyses, Is.True); + } + [Test] public void AnalyzeWord_CanAnalyze_ReturnsCorrectAnalysis() { diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs new file mode 100644 index 000000000..599cd0a47 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProcessStartInfoConstructionGateTests.cs @@ -0,0 +1,94 @@ +using System.Text.RegularExpressions; +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// A child process spawned while Xplat Code Coverage is attached inherits its CLR-profiler +/// environment variables unless something strips them first; a profiled child that then crashes or +/// is killed has been observed to destabilize the coverage channel the parent test host shares, +/// producing a trace-less test host crash. ChildProcessEnvironment.CreateStartInfo is the one +/// place that strips them, so this gate fails, naming the file, if any other source file under the +/// conformance library or its test project constructs a ProcessStartInfo directly. +/// +[TestFixture] +public sealed class ProcessStartInfoConstructionGateTests +{ + // Tolerates a namespace-qualified spelling of the type name too; a plain substring search on the + // unqualified name missed that spelling on first falsification. + private static readonly Regex RawConstruction = new( + @"new\s+(?:[\w.]+\.)?ProcessStartInfo\b", + RegexOptions.Compiled + ); + private const string FactoryFileName = "ChildProcessEnvironment.cs"; + + // This file's own doc comment and assertion messages necessarily describe the pattern in prose, + // which the regex above would otherwise flag as a violation of itself. + private const string SelfFileName = nameof(ProcessStartInfoConstructionGateTests) + ".cs"; + + private static string RepositoryRoot() + { + string? directory = TestContext.CurrentContext.TestDirectory; + while (directory is not null) + { + // The directory holding Machine.sln; the walk reads only source files, so it needs no suite. + if (File.Exists(Path.Combine(directory, "Machine.sln"))) + return directory; + directory = Directory.GetParent(directory)?.FullName; + } + + Assert.Fail("Could not locate the repository root."); + return string.Empty; + } + + private static IReadOnlyList SourceFilesUnder(string relativeRoot) + { + string root = Path.Combine(RepositoryRoot(), relativeRoot); + Assert.That(Directory.Exists(root), Is.True, $"expected a source directory at '{root}'"); + return Directory + .GetFiles(root, "*.cs", SearchOption.AllDirectories) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}")) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}")) + .ToArray(); + } + + [Test] + public void OnlyChildProcessEnvironmentConstructsAProcessStartInfo() + { + IReadOnlyList files = SourceFilesUnder( + Path.Combine("src", "SIL.Machine.Morphology.HermitCrab.Conformance") + ) + .Concat(SourceFilesUnder(Path.Combine("tests", "SIL.Machine.Morphology.HermitCrab.Tests"))) + .ToArray(); + + // A wrong root or a typo'd search pattern would make every assertion below pass vacuously by + // finding nothing to complain about, so first prove the walk actually reached real content. + Assert.That(files.Count, Is.GreaterThan(50), "the source walk found suspiciously few .cs files"); + string[] factoryFiles = files.Where(path => Path.GetFileName(path) == FactoryFileName).ToArray(); + Assert.That(factoryFiles, Has.Length.EqualTo(1), $"expected exactly one {FactoryFileName} on the walk"); + Assert.That( + RawConstruction.IsMatch(File.ReadAllText(factoryFiles[0])), + Is.True, + $"{FactoryFileName} itself no longer matches the construction pattern this gate looks for" + ); + + List violations = new(); + foreach (string file in files) + { + string name = Path.GetFileName(file); + if (name == FactoryFileName || name == SelfFileName) + continue; + if (RawConstruction.IsMatch(File.ReadAllText(file))) + violations.Add(file); + } + + Assert.That( + violations, + Is.Empty, + "these files construct a ProcessStartInfo directly instead of through " + + "ChildProcessEnvironment.CreateStartInfo, so a coverage-collector profiler attachment " + + $"is not stripped before the child spawns:{Environment.NewLine}" + + string.Join(Environment.NewLine, violations) + ); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs new file mode 100644 index 000000000..485182df4 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ProposePatchWriterTests.cs @@ -0,0 +1,37 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ProposePatchWriterTests +{ + [Test] + public void ProposeForAWordTheOracleSkipsSuggestsExpectSkip() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/loader-pattern-shapes"); + // "bit" is a declared expect_skip word. Declaring a parse for it instead makes self-check + // see a mismatch, and the oracle still skips it. + WordEntry word = fixture.Words.Words.Single(w => w.Word == "bit"); + word.ExpectSkip = false; + word.Parses.Add(new ParseEntry { Signature = "deliberately-wrong" }); + + var proposal = new StringWriter(); + Runner.RunSelfCheck( + new List { fixture }, + includePathological: true, + new HashSet { "phonology", MorpherSettings.Capability }, + propose: true, + proposal + ); + + Assert.Multiple(() => + { + Assert.That(proposal.ToString(), Does.Contain("word 'bit' signature mismatch")); + Assert.That(proposal.ToString(), Does.Contain("expect_skip: true")); + Assert.That(proposal.ToString(), Does.Not.Contain("expect_fail: true")); + }); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj index c131a8b81..822610d31 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SIL.Machine.Morphology.HermitCrab.Tests.csproj @@ -36,6 +36,7 @@ + diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs new file mode 100644 index 000000000..3d85747a6 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/StressFixtureParityTests.cs @@ -0,0 +1,32 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +[Explicit("Timing and optimization fixtures; run only with --filter TestCategory=Stress")] +[Category("Stress")] +public sealed class StressFixtureParityTests +{ + private static string StressRoot() => Path.Combine(ConformanceSuite.Root(), "stress"); + + private static IEnumerable FixtureIds() => Fixture.DiscoverAll(StressRoot()).Select(fixture => fixture.Id); + + [TestCaseSource(nameof(FixtureIds))] + public void ParsesMatchTheMasterEngine(string fixtureId) + { + List fixtures = Fixture.DiscoverAll(StressRoot()).Where(f => f.Id == fixtureId).ToList(); + RunReport report = Runner.RunSelfCheck( + fixtures, + includePathological: true, + engineCapabilities: Runner.DefaultCapabilities, + propose: false, + proposeOutput: TextWriter.Null + ); + string reasons = string.Join("; ", report.Results.Select(r => r.Reason)); + Assert.That(report.Failed, Is.EqualTo(0), reasons); + // A fixture with variants reports one result per variant, so pass means every run passed. + Assert.That(report.Results, Is.Not.Empty, reasons); + Assert.That(report.Passed, Is.EqualTo(report.Results.Count), reasons); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs new file mode 100644 index 000000000..4af6472b3 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/SuitePinCheckTests.cs @@ -0,0 +1,104 @@ +using System.Diagnostics; +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class SuitePinCheckTests +{ + private string _root = ""; + private string _suite = ""; + private string _head = ""; + private string? _override; + + [SetUp] + public void SetUp() + { + _override = Environment.GetEnvironmentVariable(ConformanceSuite.OverrideVariable); + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, null); + _root = Path.Combine(Path.GetTempPath(), $"pin-check-{Guid.NewGuid():N}"); + _suite = Path.Combine(_root, "conformance"); + Directory.CreateDirectory(Path.Combine(_suite, "stress")); + File.WriteAllText(Path.Combine(_suite, "constructs.txt"), "a\n"); + Git(_suite, "init", "-q"); + Git(_suite, "add", "."); + Git( + _suite, + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "x" + ); + _head = Git(_suite, "rev-parse", "HEAD").Trim(); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, _override); + foreach (string path in Directory.EnumerateFileSystemEntries(_root, "*", SearchOption.AllDirectories)) + File.SetAttributes(path, FileAttributes.Normal); + Directory.Delete(_root, recursive: true); + } + + [Test] + public void Warnings_ForAnUntouchedCloneAtThePin_AreEmpty() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), _head + "\n"); + + Assert.That(SuitePinCheck.Warnings(_suite), Is.Empty); + Assert.That(SuitePinCheck.Warnings(Path.Combine(_suite, "stress")), Is.Empty); + } + + [Test] + public void Warnings_ForACloneAtAnotherCommit_SayTheSuiteIsStale() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), new string('a', 40) + "\n"); + + Assert.That(SuitePinCheck.Warnings(_suite), Has.Exactly(1).Contain("stale suite")); + } + + [Test] + public void Warnings_ForAnUntrackedFixtureAtThePin_NameIt() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), _head + "\n"); + File.WriteAllText(Path.Combine(_suite, "extra.txt"), "x"); + + Assert.That(SuitePinCheck.Warnings(_suite), Has.Exactly(1).Contain("extra.txt")); + } + + [Test] + public void Warnings_WhenTheOverrideIsSetOrTheFixturesAreElsewhere_AreEmpty() + { + File.WriteAllText(Path.Combine(_root, "conformance.commit"), new string('a', 40) + "\n"); + string elsewhere = Path.Combine(_root, "other"); + Directory.CreateDirectory(elsewhere); + Assert.That(SuitePinCheck.Warnings(elsewhere), Is.Empty); + + Environment.SetEnvironmentVariable(ConformanceSuite.OverrideVariable, _suite); + Assert.That(SuitePinCheck.Warnings(_suite), Is.Empty); + } + + private static string Git(string workingDirectory, params string[] arguments) + { + ProcessStartInfo startInfo = ChildProcessEnvironment.CreateStartInfo("git"); + startInfo.WorkingDirectory = workingDirectory; + startInfo.RedirectStandardOutput = true; + foreach (string variable in new[] { "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE" }) + startInfo.EnvironmentVariables.Remove(variable); + foreach (string argument in arguments) + startInfo.ArgumentList.Add(argument); + using Process process = Process.Start(startInfo)!; + string output = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + Assert.That(process.ExitCode, Is.EqualTo(0), $"git {string.Join(' ', arguments)}"); + return output; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs new file mode 100644 index 000000000..05de67bd7 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/ToolScriptExitCodeTests.cs @@ -0,0 +1,50 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class ToolScriptExitCodeTests +{ + [Test] + public void Main_WithAScriptWhoseFirstCommandFails_ExitsWithThatFailureEvenWhenALaterCommandSucceeds() + { + (int failing, int passing) = RunScripts(["batch --start=x {words} {out}", "batch {words} {out}"]); + + Assert.Multiple(() => + { + Assert.That(failing, Is.Not.EqualTo(0)); + Assert.That(passing, Is.EqualTo(0)); + }); + } + + private static (int Failing, int Passing) RunScripts(string[] failingScript) + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + string directory = Path.Combine(Path.GetTempPath(), $"tool-script-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + string words = Path.Combine(directory, "words.txt"); + File.WriteAllLines(words, ["a"]); + string output = Path.Combine(directory, "out.tsv"); + TextWriter originalOut = Console.Out; + try + { + Console.SetOut(TextWriter.Null); + int Run(string[] lines) + { + string script = Path.Combine(directory, $"script-{Guid.NewGuid():N}.txt"); + File.WriteAllLines( + script, + lines.Select(line => line.Replace("{words}", words).Replace("{out}", output)) + ); + return Program.Main(["-i", fixture.GrammarPath, "-s", script]); + } + return (Run(failingScript), Run(["batch {words} {out}"])); + } + finally + { + Console.SetOut(originalOut); + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs new file mode 100644 index 000000000..36fc723e8 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/WordsYamlMorpherSettingsTests.cs @@ -0,0 +1,196 @@ +using NUnit.Framework; +using SIL.Machine.Morphology.HermitCrab.Conformance; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public sealed class WordsYamlMorpherSettingsTests +{ + private const string Words = + "words:\n - word: kat\n note: n\n parses:\n - signature: \"KAT|kat\"\n rules: []\n"; + + private static WordsYaml Load(string frontMatter) + { + string path = Path.Combine(Path.GetTempPath(), "words-" + Guid.NewGuid().ToString("N") + ".yaml"); + if ( + !frontMatter.Contains("requires:", StringComparison.Ordinal) + && ( + frontMatter.Contains("morpher:", StringComparison.Ordinal) + || frontMatter.Contains("variants:", StringComparison.Ordinal) + ) + ) + { + frontMatter = "requires: [morpher-settings]\n" + frontMatter; + } + File.WriteAllText(path, "language: T\n" + frontMatter + Words); + try + { + return WordsYamlLoader.Load(path); + } + finally + { + File.Delete(path); + } + } + + [Test] + public void AbsentBlockMeansTheDefaults() + { + WordsYaml words = Load(""); + Assert.That(words.Morpher.IsEmpty, Is.True); + } + + [Test] + public void EveryAcceptedKeyIsRead() + { + WordsYaml words = Load( + "morpher:\n DelReapps: 1\n MaxRoots: 3\n MergeAnalyses: false\n MakePartialsFinal: true\n GuessRoots: false\n" + ); + Assert.Multiple(() => + { + Assert.That(words.Morpher.DelReapps, Is.EqualTo(1)); + Assert.That(words.Morpher.MaxRoots, Is.EqualTo(3)); + Assert.That(words.Morpher.MergeAnalyses, Is.False); + Assert.That(words.Morpher.MakePartialsFinal, Is.True); + Assert.That(words.Morpher.GuessRoots, Is.False); + Assert.That(words.Morpher.GuessRoots, Is.EqualTo(false)); + }); + } + + [TestCase("morpher:\n MaxAlternatives: 5\n", "unknown key 'MaxAlternatives'")] + [TestCase("morpher:\n DelReapps: one\n", "DelReapps")] + [TestCase("morpher:\n DelReapps: -1\n", "DelReapps")] + [TestCase("morpher:\n DelReapps: 2147483648\n", "DelReapps")] + [TestCase("morpher:\n MaxRoots: 0\n", "MaxRoots")] + [TestCase("morpher:\n MaxRoots: 2147483648\n", "MaxRoots")] + [TestCase("morpher:\n MergeAnalyses: yes\n", "expected 'true' or 'false'")] + [TestCase("morpher: {}\n", "empty block")] + [TestCase("morpher: 1\n", "expected a mapping")] + public void AnUnappliableBlockIsAnError(string frontMatter, string expected) + { + var ex = Assert.Throws(() => Load(frontMatter)); + Assert.That(ex!.Message, Does.Contain(expected)); + } + + [Test] + public void TheFactoryAppliesEveryDeclaredValueAndOnlyThose() + { + Fixture fixture = Fixture.DiscoverAll(ConformanceSuite.Root()).First(); + Language language = XmlLanguageLoader.Load(fixture.GrammarPath); + var settings = new MorpherSettings { DelReapps = 2, MakePartialsFinal = true }; + + Morpher morpher = ConformanceMorpherFactory.Create(language, settings); + + Assert.Multiple(() => + { + Assert.That(morpher.DeletionReapplications, Is.EqualTo(2)); + Assert.That(morpher.AlwaysEnforceFinalTemplates, Is.True); + Assert.That(morpher.MaxStemCount, Is.EqualTo(2), "undeclared: constructor default"); + Assert.That(morpher.MergeEquivalentAnalyses, Is.True, "undeclared: constructor default"); + }); + } + + [Test] + public void VariantsAreFullRunsAndRequireTheSameOrderedWords() + { + WordsYaml words = Load( + "morpher:\n DelReapps: 1\nvariants:\n - name: roots-two\n morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n" + ); + Assert.Multiple(() => + { + Assert.That(words.Runs.Select(run => run.Name), Is.EqualTo(new[] { "default", "roots-two" })); + Assert.That(words.Variants.Single().Morpher.MaxRoots, Is.EqualTo(2)); + Assert.That( + words.Variants.Single().Morpher.DelReapps, + Is.Null, + "variant settings start from engine defaults" + ); + Assert.That(words.Variants.Single().Words.Single().Note, Is.EqualTo("n")); + }); + } + + [TestCase("default", "default is reserved")] + [TestCase("BadName", "must match")] + public void InvalidVariantNamesAreRejected(string name, string expected) + { + string yaml = + $"requires: [morpher-settings]\nvariants:\n - name: {name}\n morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n"; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain(expected)); + } + + [Test] + public void DuplicateVariantNamesAreRejected() + { + const string VariantBody = + " morpher:\n MaxRoots: 2\n words:\n - word: kat\n note: n\n parses:\n - signature: KAT|kat\n rules: []\n"; + string yaml = + "requires: [morpher-settings]\nvariants:\n - name: same\n" + + VariantBody + + " - name: same\n" + + VariantBody; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain("duplicate variant name")); + } + + [Test] + public void VariantWordOrderMustMatchDefault() + { + string yaml = + "requires: [morpher-settings]\nvariants:\n - name: changed\n morpher:\n MaxRoots: 2\n words:\n - word: other\n note: n\n parses:\n - signature: OTHER|other\n rules: []\n"; + var ex = Assert.Throws(() => Load(yaml)); + Assert.That(ex!.Message, Does.Contain("same order")); + } + + [Test] + public void SelfCheckRunsVariantsAndFailsAWrongVariantExpectationByName() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/deletion-reapplications"); + WordRun variant = fixture.Words.Variants.Single(v => v.Name == "del-reapps-two"); + variant.Words[0].Parses[0].Signature = "deliberately-wrong"; + + RunReport report = Runner.RunSelfCheck( + new List { fixture }, + includePathological: true, + new HashSet { "phonology", MorpherSettings.Capability }, + propose: false, + TextWriter.Null + ); + + FixtureResult failed = report.Results.Single(r => + r.FixtureId.Contains("variant del-reapps-two", StringComparison.Ordinal) + ); + Assert.That(failed.Outcome, Is.EqualTo(FixtureOutcome.Failed)); + Assert.That(failed.Reason, Does.Contain("mismatched")); + } + + [Test] + public void ABlockIsACapabilityRequirement() + { + Fixture fixture = Fixture + .DiscoverAll(ConformanceSuite.Root()) + .Single(f => f.Id == "edge-cases/deletion-reapplications"); + + Assert.That( + RequiresDerivation.Derive(fixture.GrammarPath, fixture.Words.Morpher), + Is.EqualTo(new[] { "phonology", MorpherSettings.Capability }) + ); + Assert.That( + RequiresDerivation.Derive(fixture.GrammarPath, MorpherSettings.Defaults), + Is.EqualTo(new[] { "phonology" }) + ); + Assert.That( + RequiresDerivation.Derive( + fixture.GrammarPath, + new[] + { + MorpherSettings.Defaults, + new MorpherSettings { MaxRoots = 2 }, + } + ), + Is.EqualTo(new[] { "phonology", MorpherSettings.Capability }) + ); + } +}