diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs index ac28d7a87..495a57587 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs @@ -32,9 +32,8 @@ public IEnumerable Apply(Word input) HashSet outputSet = tempSet; outputSet.Clear(); - // Limit alternatives accross all invocations of _rules[i].Apply. + // The limit spans every input to this stratum in this parse. int alternativeCount = 0; - _rules[i].MaxAlternatives = _morpher.MaxAlternatives; foreach (Word inData in inputSet) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index ef48637ac..6431ff248 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -16,7 +16,6 @@ internal class AnalysisStratumRule : IRule private readonly RuleBatch _templatesRule; private readonly Stratum _stratum; private readonly Morpher _morpher; - private int _maxAlternatives; public AnalysisStratumRule(Morpher morpher, Stratum stratum) { @@ -30,45 +29,47 @@ public AnalysisStratumRule(Morpher morpher, Stratum stratum) false, FreezableEqualityComparer.Default ); - _mrulesRule = null; - IEnumerable> mrules = stratum - .MorphologicalRules.Select(mrule => CompileMorphologicalRule(mrule, morpher)) - .Reverse(); - switch (stratum.MorphologicalRuleOrder) + _mrulesRule = CreateMorphologicalRules( + stratum.MorphologicalRules.Select(mrule => CompileMorphologicalRule(mrule, morpher)).Reverse() + ); + } + + private RuleCascade CreateMorphologicalRules(IEnumerable> rules) + { + switch (_stratum.MorphologicalRuleOrder) { case MorphologicalRuleOrder.Linear: // Use PermutationRuleCascade instead of LinearRuleCascade // because morphological rules should be considered optional // during unapplication (they are obligatory during application, // but we don't know they have been applied during unapplication). - _mrulesRule = new PermutationRuleCascade( - mrules, + return new PermutationRuleCascade( + rules, true, FreezableEqualityComparer.Default ); - break; case MorphologicalRuleOrder.Unordered: - if (morpher.MaxDegreeOfParallelism == 1) + if (_morpher.MaxDegreeOfParallelism == 1) { - _mrulesRule = new CombinationRuleCascade( - mrules, + return new CombinationRuleCascade( + rules, true, FreezableEqualityComparer.Default ); } else { - _mrulesRule = new ParallelCombinationRuleCascade( - mrules, + return new ParallelCombinationRuleCascade( + rules, true, FreezableEqualityComparer.Default ) { - MaxDegreeOfParallelism = morpher.MaxDegreeOfParallelism, + MaxDegreeOfParallelism = _morpher.MaxDegreeOfParallelism, }; } - break; } + return null; } private IRule CompileAffixTemplate(AffixTemplate template, Morpher morpher) @@ -107,25 +108,34 @@ private IRule CompilePhonologicalRule(IPhonologicalRule prule, } } - public int MaxAlternatives - { - get { return _maxAlternatives; } - set - { - _maxAlternatives = value; - _mrulesRule.MaxAlternatives = value; - _templatesRule.MaxAlternatives = value; - } - } - public IEnumerable Apply(Word input) { + // Standalone stratum application is uncapped. int alternativeCount = 0; - return Apply(input, ref alternativeCount); + return Apply(input, ref alternativeCount, false); } internal IEnumerable Apply(Word input, ref int alternativeCount) { + return Apply(input, ref alternativeCount, true); + } + + private IEnumerable Apply(Word input, ref int alternativeCount, bool enforceLimit) + { + int maxAlternatives = enforceLimit ? _morpher.MaxAlternatives : 0; + RuleCascade morphologicalRules = _mrulesRule; + RuleBatch templates = _templatesRule; + if (maxAlternatives > 0) + { + // Limits belong to this invocation; compiled child rules remain shared and read-only. + morphologicalRules = CreateMorphologicalRules(_mrulesRule.Rules); + morphologicalRules.MaxAlternatives = maxAlternatives; + templates = new RuleBatch(_templatesRule.Rules, false, _templatesRule.Comparer) + { + MaxAlternatives = maxAlternatives, + }; + } + if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.BeginUnapplyStratum(_stratum, input); @@ -144,7 +154,8 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) wordCache = new Dictionary(); // AnalysisStratumRule.Apply should cover the inverse of SynthesisStratumRule.Apply. - IEnumerable mruleOutWords = ApplyTemplates(input).Concat(ApplyMorphologicalRules(input)); + IEnumerable mruleOutWords = ApplyTemplates(input, morphologicalRules, templates) + .Concat(ApplyMorphologicalRules(input, morphologicalRules, templates)); Debug.Assert(mruleOutWords != null); var output = new HashSet(FreezableEqualityComparer.Default) { input }; @@ -153,7 +164,7 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) foreach (Word mruleOutWord in mruleOutWords) { alternativeCount++; - if (_maxAlternatives > 0 && alternativeCount > _maxAlternatives) + if (maxAlternatives > 0 && alternativeCount > maxAlternatives) { // Stops before full enumeration because ApplyTemplates and ApplyMorphologicalRules use yield return. throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); @@ -182,9 +193,15 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) return output; } - private IEnumerable ApplyMorphologicalRules(Word input) + private IEnumerable ApplyMorphologicalRules( + Word input, + RuleCascade morphologicalRules, + RuleBatch templates + ) { - foreach (Word mruleOutWord in _mrulesRule.Apply(input).Distinct(FreezableEqualityComparer.Default)) + foreach ( + Word mruleOutWord in morphologicalRules.Apply(input).Distinct(FreezableEqualityComparer.Default) + ) { switch (_stratum.MorphologicalRuleOrder) { @@ -193,7 +210,7 @@ private IEnumerable ApplyMorphologicalRules(Word input) break; case MorphologicalRuleOrder.Unordered: - foreach (Word tempOutWord in ApplyTemplates(mruleOutWord)) + foreach (Word tempOutWord in ApplyTemplates(mruleOutWord, morphologicalRules, templates)) yield return tempOutWord; yield return mruleOutWord; break; @@ -201,14 +218,18 @@ private IEnumerable ApplyMorphologicalRules(Word input) } } - private IEnumerable ApplyTemplates(Word input) + private IEnumerable ApplyTemplates( + Word input, + RuleCascade morphologicalRules, + RuleBatch templates + ) { - foreach (Word tempOutWord in _templatesRule.Apply(input).Distinct(FreezableEqualityComparer.Default)) + foreach (Word tempOutWord in templates.Apply(input).Distinct(FreezableEqualityComparer.Default)) { switch (_stratum.MorphologicalRuleOrder) { case MorphologicalRuleOrder.Linear: - foreach (Word outWord in ApplyMorphologicalRules(tempOutWord)) + foreach (Word outWord in ApplyMorphologicalRules(tempOutWord, morphologicalRules, templates)) yield return outWord; if (!FreezableEqualityComparer.Default.Equals(input, tempOutWord)) yield return tempOutWord; @@ -217,7 +238,9 @@ private IEnumerable ApplyTemplates(Word input) case MorphologicalRuleOrder.Unordered: if (!FreezableEqualityComparer.Default.Equals(input, tempOutWord)) { - foreach (Word outWord in ApplyMorphologicalRules(tempOutWord)) + foreach ( + Word outWord in ApplyMorphologicalRules(tempOutWord, morphologicalRules, templates) + ) yield return outWord; yield return tempOutWord; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 41f1a964e..7c1dda6c8 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -32,7 +32,11 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator private readonly IList _lexicalPatterns = new List(); public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0) + : this(traceManager, lang, maxDegreeOfParallelism, 0) { } + + public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism, int maxAlternatives) { + MaxAlternatives = maxAlternatives; _lang = lang; _traceManager = traceManager; // Must be set before CompileAnalysisRule: AnalysisStratumRule picks a sequential vs. parallel @@ -60,7 +64,6 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle _analysisRule = lang.CompileAnalysisRule(this); _synthesisRule = lang.CompileSynthesisRule(this); MaxStemCount = 2; - MaxAlternatives = 0; MergeEquivalentAnalyses = true; PruneDisagreeingCopies = true; LexEntrySelector = entry => true; @@ -93,8 +96,9 @@ public ITraceManager TraceManager /// /// MaxAlternatives limits the number of alternatives considered. /// If the limit is exceeded, then MaxAlternativesExceededException is thrown. + /// Set this value with the constructor. /// - public int MaxAlternatives { get; set; } + public int MaxAlternatives { get; } /// /// Merge analyses with equivalent state for every analysis-side rule. @@ -128,6 +132,7 @@ public ITraceManager TraceManager /// (a value of 1 compiles the sequential cascade for unordered strata). It must remain a performance /// knob only: nothing that changes which analyses a parse returns, or whether /// is exceeded, may depend on it. + /// throws unless this value is 1. /// public int MaxDegreeOfParallelism { get; } @@ -225,6 +230,88 @@ public IEnumerable ParseWord(string word, out object trace, bool guessRoot return syntheses; } + /// + /// Eagerly parses surface forms in input order, capturing each parse's exception and trace in its result. + /// Requires a Morpher constructed with maxDegreeOfParallelism: 1. Tracing runs the batch serially. + /// Keep the Language, settings, and tracing fixed until completion; selectors and custom rules must + /// support concurrent calls when maxWordConcurrency is greater than 1 and tracing is disabled. + /// Do not overlap traced calls sharing a trace manager. Failed parses have empty analyses. + /// Progress receives the input index and surface form, with a null result at the start and the + /// completed result at the finish. It runs on parsing threads (the calling thread for serial batches), + /// possibly concurrently, and must be fast and thread-safe. Callback exceptions are collected and + /// propagated as an AggregateException after all words and both notifications have been processed. + /// Batch argument validation and input enumeration errors propagate to the caller. + /// + public IReadOnlyList ParseWords( + IEnumerable words, + int maxWordConcurrency = 4, + bool guessRoot = false, + Action progress = null + ) + { + if (words == null) + throw new ArgumentNullException(nameof(words)); + if (maxWordConcurrency < 1) + throw new ArgumentOutOfRangeException(nameof(maxWordConcurrency)); + if (MaxDegreeOfParallelism != 1) + { + throw new InvalidOperationException( + "ParseWords requires a Morpher constructed with maxDegreeOfParallelism: 1." + ); + } + + string[] inputs = words.ToArray(); + var results = new WordParseResult[inputs.Length]; + var callbackErrors = new ConcurrentQueue(); + void Notify(int index, WordParseResult result) + { + try + { + progress?.Invoke(index, inputs[index], result); + } + catch (Exception error) + { + callbackErrors.Enqueue(error); + } + } + + void Parse(int index) + { + object trace = null; + Notify(index, null); + try + { + var analyses = ParseWord(inputs[index], out trace, guessRoot).ToList().AsReadOnly(); + results[index] = new WordParseResult(inputs[index], analyses, trace, null); + } + catch (Exception error) + { + results[index] = new WordParseResult(inputs[index], Array.Empty(), trace, error); + } + Notify(index, results[index]); + } + + if (_traceManager.IsTracing || maxWordConcurrency == 1) + { + for (int i = 0; i < inputs.Length; i++) + Parse(i); + } + else + { + Parallel.For( + 0, + inputs.Length, + new ParallelOptions { MaxDegreeOfParallelism = maxWordConcurrency }, + Parse + ); + } + + if (!callbackErrors.IsEmpty) + throw new AggregateException("One or more progress callbacks failed.", callbackErrors); + + return Array.AsReadOnly(results); + } + /// /// Generates surface forms from the specified word synthesis information. /// diff --git a/src/SIL.Machine.Morphology.HermitCrab/WordParseResult.cs b/src/SIL.Machine.Morphology.HermitCrab/WordParseResult.cs new file mode 100644 index 000000000..0ffc2c892 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/WordParseResult.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab +{ + public sealed class WordParseResult + { + internal WordParseResult(string surfaceForm, IReadOnlyList analyses, object trace, Exception error) + { + SurfaceForm = surfaceForm; + Analyses = analyses; + Trace = trace; + Error = error; + } + + public string SurfaceForm { get; } + + public IReadOnlyList Analyses { get; } + + public object Trace { get; } + + public Exception Error { get; } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs index d9149d058..718318386 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs @@ -76,8 +76,7 @@ public void AnalyzeWord_MaxAlternatives() ); Morphophonemic.MorphologicalRules.Add(gSuffix); - var morpher = new Morpher(TraceManager, Language); - morpher.MaxAlternatives = 1; + var morpher = new Morpher(TraceManager, Language, 0, 1); Assert.Throws(() => morpher.AnalyzeWord("sagd")); } @@ -647,6 +646,422 @@ public void ParseWord_SingleThreaded_MatchesParallel_OnLinearStratumWithAffixTem Assert.That(parallel.ParseWord(word), Is.Not.Empty, $"'{word}' must parse"); } + [TestCase(MorphologicalRuleOrder.Unordered, 0)] + [TestCase(MorphologicalRuleOrder.Unordered, 100)] + [TestCase(MorphologicalRuleOrder.Linear, 0)] + [TestCase(MorphologicalRuleOrder.Linear, 100)] + public void ParseWords_PreservesInputOrderAndSingleWordAnalysisSets(MorphologicalRuleOrder ruleOrder, int limit) + { + AddAffixTemplateAndCommutingPrefixes(); + SetRuleOrder(ruleOrder); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1, maxAlternatives: limit); + string[] words = { "gudisagd", "sag", "digusagd", "sagt", "disagd", "sag" }; + string[][] expected = words + .Select(w => morpher.ParseWord(w).Select(BatchAnalysisSignature).ToArray()) + .ToArray(); + + foreach (int concurrency in new[] { 1, 4 }) + { + IReadOnlyList results = morpher.ParseWords(words, concurrency); + + Assert.That(results.Select(r => r.SurfaceForm), Is.EqualTo(words)); + for (int i = 0; i < words.Length; i++) + { + Assert.That(results[i].Error, Is.Null); + Assert.That(results[i].Trace, Is.Null); + Assert.That(results[i].Analyses.Select(BatchAnalysisSignature), Is.EquivalentTo(expected[i])); + } + Assert.That(results[1].Analyses, Is.Not.Empty); + } + } + + [Test] + public void MaxAlternatives_HasNoSetter() + { + Assert.That(typeof(Morpher).GetProperty(nameof(Morpher.MaxAlternatives))!.SetMethod, Is.Null); + } + + [TestCase(MorphologicalRuleOrder.Unordered, 0)] + [TestCase(MorphologicalRuleOrder.Unordered, 1)] + [TestCase(MorphologicalRuleOrder.Unordered, 2)] + [TestCase(MorphologicalRuleOrder.Linear, 0)] + [TestCase(MorphologicalRuleOrder.Linear, 1)] + [TestCase(MorphologicalRuleOrder.Linear, 2)] + public void ParseWord_MaxAlternativesIsFixedForEachMorpher(MorphologicalRuleOrder ruleOrder, int degree) + { + AddAffixTemplateAndCommutingPrefixes(); + SetRuleOrder(ruleOrder); + var morpher = new Morpher(TraceManager, Language, degree); + string[] expected = morpher.ParseWord("gudisagd").Select(BatchAnalysisSignature).ToArray(); + Assert.That(expected, Is.Not.Empty); + + foreach (int limit in new[] { 1, 100, 1, 0 }) + { + morpher = new Morpher(TraceManager, Language, degree, limit); + Assert.That(morpher.MaxAlternatives, Is.EqualTo(limit)); + if (limit == 1) + Assert.Throws(() => morpher.ParseWord("gudisagd")); + else + Assert.That(morpher.ParseWord("gudisagd").Select(BatchAnalysisSignature), Is.EquivalentTo(expected)); + } + } + + [Test] + public void CompileAnalysisRule_DirectStratumInvocationRemainsUncapped() + { + AddAffixTemplateAndCommutingPrefixes(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1, maxAlternatives: 1); + IRule rule = Morphophonemic.CompileAnalysisRule(morpher); + var input = new Word(Morphophonemic, Table3.Segment("gudisagd")); + input.Freeze(); + + Assert.That(rule.Apply(input), Is.Not.Empty); + Assert.Throws(() => morpher.ParseWord("gudisagd")); + } + + [Test] + public void ParseWords_MatchesSingleWordAnalysisSetsWithCompounding() + { + AddCompoundingAndPrefixRules(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + string[] words = { "pʰutdidat", "pʰutdat", "pʰutdat" }; + + foreach (int concurrency in new[] { 1, 4 }) + { + IReadOnlyList results = morpher.ParseWords(words, concurrency); + Assert.That(results.Select(r => r.SurfaceForm), Is.EqualTo(words)); + for (int i = 0; i < words.Length; i++) + { + Assert.That(results[i].Error, Is.Null); + Assert.That( + results[i].Analyses.Select(BatchAnalysisSignature), + Is.EquivalentTo(morpher.ParseWord(words[i]).Select(BatchAnalysisSignature)) + ); + } + Assert.That(results[1].Analyses, Has.Count.EqualTo(2)); + } + } + + [TestCase(2)] + [TestCase(4)] + public void ParseWords_UsesBoundedWordConcurrency(int concurrency) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + using var entered = new CountdownEvent(concurrency); + int calls = 0; + int active = 0; + int peak = 0; + object sync = new(); + morpher.LexEntrySelector = _ => + { + int current = Interlocked.Increment(ref active); + try + { + lock (sync) + peak = Math.Max(peak, current); + if (Interlocked.Increment(ref calls) <= concurrency) + entered.Signal(); + if (!entered.Wait(TimeSpan.FromSeconds(10))) + throw new TimeoutException("The requested word workers did not enter lexical lookup."); + return true; + } + finally + { + Interlocked.Decrement(ref active); + } + }; + string[] words = Enumerable.Repeat("sag", 8).ToArray(); + + IReadOnlyList results = + concurrency == 4 ? morpher.ParseWords(words) : morpher.ParseWords(words, concurrency); + + Assert.That(results.Select(r => r.Error), Is.All.Null); + Assert.That(results.Select(r => r.Analyses.Count), Is.All.GreaterThan(0)); + Assert.That(peak, Is.EqualTo(concurrency)); + } + + [TestCase(1)] + [TestCase(2)] + [TestCase(4)] + public void ParseWords_ReportsProgressInOrderWithinConcurrencyLimit(int concurrency) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + string[] words = { "sag", "sag$", "sagt", "sag", "sag", "sag" }; + var events = new List<(int Index, string Word, WordParseResult Result)>(); + object sync = new(); + int active = 0; + int peak = 0; + IReadOnlyList results = morpher.ParseWords( + words, + concurrency, + progress: (index, word, result) => + { + lock (sync) + { + events.Add((index, word, result)); + active += result == null ? 1 : -1; + peak = Math.Max(peak, active); + } + } + ); + + Assert.That(events, Has.Count.EqualTo(words.Length * 2)); + Assert.That(active, Is.Zero); + Assert.That(peak, Is.InRange(1, concurrency)); + for (int i = 0; i < words.Length; i++) + { + var perWord = events.Where(e => e.Index == i).ToArray(); + Assert.That(perWord, Has.Length.EqualTo(2)); + Assert.That(perWord.Select(e => e.Word), Is.EqualTo(new[] { words[i], words[i] })); + Assert.That(perWord[0].Result, Is.Null); + Assert.That(perWord[1].Result, Is.SameAs(results[i])); + } + Assert.That(results[1].Error, Is.TypeOf()); + } + + [TestCase(1)] + [TestCase(4)] + public void ParseWords_PropagatesCallbackErrorsAfterAllWordsFinish(int concurrency) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + string[] words = { "sag", "sag$", "sag", "sag" }; + var completed = new WordParseResult[words.Length]; + var starts = new int[words.Length]; + var finishes = new int[words.Length]; + var error = Assert.Throws(() => + morpher.ParseWords( + words, + concurrency, + progress: (index, word, result) => + { + if (result == null) + { + Interlocked.Increment(ref starts[index]); + if (index == 0) + throw new InvalidOperationException("start"); + } + else + { + completed[index] = result; + Interlocked.Increment(ref finishes[index]); + if (index == 2) + throw new InvalidOperationException("finish"); + } + } + ) + ); + + Assert.That(error.InnerExceptions.Select(e => e.Message), Is.EquivalentTo(new[] { "start", "finish" })); + Assert.That(starts, Is.All.EqualTo(1)); + Assert.That(finishes, Is.All.EqualTo(1)); + Assert.That(completed[1].Error, Is.TypeOf()); + foreach (int index in new[] { 0, 2, 3 }) + { + Assert.That(completed[index].Error, Is.Null); + Assert.That(completed[index].Analyses, Is.Not.Empty); + } + } + + [TestCase(1)] + [TestCase(4)] + public void ParseWords_CapturesInvalidShapesWithoutLosingOtherResults(int concurrency) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + string[] words = { "sag", "sag$", "sagt", "sag" }; + + IReadOnlyList results = morpher.ParseWords(words, concurrency); + + Assert.That(results.Select(r => r.SurfaceForm), Is.EqualTo(words)); + Assert.That(results[1].Error, Is.TypeOf()); + Assert.That(results[1].Analyses, Is.Empty); + Assert.That(results[1].Trace, Is.Null); + Assert.That(results[2].Error, Is.Null); + Assert.That(results[2].Analyses, Is.Empty); + foreach (int index in new[] { 0, 3 }) + { + Assert.That(results[index].Error, Is.Null); + Assert.That(results[index].Analyses, Is.Not.Empty); + } + } + + [TestCase(1)] + [TestCase(4)] + public void ParseWords_CapturesMaxAlternativesPerWord(int concurrency) + { + AddAffixTemplateAndCommutingPrefixes(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1, maxAlternatives: 1); + + IReadOnlyList results = morpher.ParseWords( + new[] { "gudisagd", "sag", "digusagd", "sag" }, + concurrency + ); + + foreach (int index in new[] { 0, 2 }) + { + Assert.That(results[index].Error, Is.TypeOf()); + Assert.That(results[index].Analyses, Is.Empty); + } + foreach (int index in new[] { 1, 3 }) + { + Assert.That(results[index].Error, Is.Null); + Assert.That(results[index].Analyses, Is.Not.Empty); + } + + Assert.That(morpher.MaxAlternatives, Is.EqualTo(1)); + Assert.That( + morpher.ParseWords(new[] { "gudisagd" }, concurrency)[0].Error, + Is.TypeOf() + ); + var unlimited = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + Assert.That(unlimited.ParseWords(new[] { "gudisagd" }, concurrency)[0].Analyses, Is.Not.Empty); + } + + [TestCase(-1)] + [TestCase(0)] + [TestCase(2)] + public void ParseWords_RequiresSequentialMorpher(int degree) + { + var morpher = new Morpher(TraceManager, Language, degree); + + var error = Assert.Throws(() => morpher.ParseWords(new[] { "sag" })); + + Assert.That(error!.Message, Does.Contain("maxDegreeOfParallelism: 1")); + Assert.That(morpher.ParseWord("sag"), Is.Not.Empty); + } + + [Test] + public void ParseWords_ValidatesBatchArguments() + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + Assert.Throws(() => morpher.ParseWords(null!)); + Assert.Throws(() => morpher.ParseWords(new[] { "sag" }, 0)); + Assert.Throws(() => morpher.ParseWords(new[] { "sag" }, -1)); + Assert.That(morpher.ParseWords(Array.Empty()), Is.Empty); + } + + [Test] + public void ParseWords_EnumeratesInputOnceAndPropagatesEnumerationErrors() + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + int enumerations = 0; + IEnumerable Inputs() + { + enumerations++; + yield return "sag"; + yield return "sag"; + } + var error = new InvalidOperationException("Input enumeration failed."); + IEnumerable BrokenInputs() + { + yield return "sag"; + throw error; + } + + Assert.That(morpher.ParseWords(Inputs()), Has.Count.EqualTo(2)); + Assert.That(enumerations, Is.EqualTo(1)); + Assert.That( + Assert.Throws(() => morpher.ParseWords(BrokenInputs())), + Is.SameAs(error) + ); + } + + [Test] + public void ParseWords_GuessesRootsWhenRequested() + { + var naturalClass = new NaturalClass(new FeatureStruct()) { Name = "BatchAny" }; + Morphophonemic.CharacterDefinitionTable.AddNaturalClass(naturalClass); + LexEntry pattern = AddEntry("batch_pattern", new FeatureStruct(), Morphophonemic, "[BatchAny]*"); + try + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + string[] words = { "gag", "sag", "gug" }; + Assert.That(morpher.ParseWords(words)[0].Analyses, Is.Empty); + + foreach (int concurrency in new[] { 1, 4 }) + { + IReadOnlyList results = morpher.ParseWords(words, concurrency, guessRoot: true); + for (int i = 0; i < words.Length; i++) + { + Assert.That(results[i].Error, Is.Null); + Assert.That(results[i].Analyses, Is.Not.Empty); + Assert.That( + results[i].Analyses.Select(BatchAnalysisSignature), + Is.EquivalentTo(morpher.ParseWord(words[i], out _, true).Select(BatchAnalysisSignature)) + ); + } + Assert.That(results[0].Analyses.All(w => w.RootAllomorph.Guessed), Is.True); + } + } + finally + { + Morphophonemic.Entries.Remove(pattern); + Entries.Remove("batch_pattern"); + } + } + + [Test] + public void ParseWords_TracingRunsOnCallingThreadAndRetainsPerWordTraces() + { + AddAffixTemplateAndCommutingPrefixes(); + TraceManager.IsTracing = true; + try + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + var error = new InvalidOperationException("First parse failed."); + int callingThread = Environment.CurrentManagedThreadId; + int offThreadCalls = 0; + int calls = 0; + morpher.RuleSelector = _ => + { + if (Environment.CurrentManagedThreadId != callingThread) + Interlocked.Increment(ref offThreadCalls); + if (Interlocked.Increment(ref calls) == 1) + throw error; + return true; + }; + + IReadOnlyList results = morpher.ParseWords(new[] { "sag", "digusagd", "sag" }, 4); + + Assert.That(offThreadCalls, Is.Zero); + Assert.That(results[0].Error, Is.SameAs(error)); + Assert.That(results[0].Analyses, Is.Empty); + Assert.That(results.Select(r => r.Trace), Is.All.InstanceOf()); + Assert.That(results.Select(r => r.Trace).Distinct().Count(), Is.EqualTo(3)); + foreach (int index in new[] { 1, 2 }) + { + Assert.That(results[index].Error, Is.Null); + Assert.That(results[index].Analyses, Is.Not.Empty); + Assert.That( + results[index].Analyses.Select(BatchAnalysisSignature), + Is.EquivalentTo(morpher.ParseWord(results[index].SurfaceForm).Select(BatchAnalysisSignature)) + ); + } + } + finally + { + TraceManager.IsTracing = false; + } + } + + private string BatchAnalysisSignature(Word word) + { + static string AllomorphKey(Allomorph allomorph) => $"{allomorph.Morpheme.Id}:{allomorph.Index}"; + return string.Join( + "|", + new[] + { + AllomorphKey(word.RootAllomorph), + string.Join(",", word.MorphologicalRules.Select(r => r.Name)), + string.Join(",", word.Allomorphs.Select(AllomorphKey).OrderBy(s => s, StringComparer.Ordinal)), + string.Join(",", word.AllomorphsInMorphOrder.Select(AllomorphKey)), + word.Shape.ToString(Language.SurfaceStratum.CharacterDefinitionTable, false), + word.SyntacticFeatureStruct.ToString(), + word.RealizationalFeatureStruct.ToString(), + } + ); + } + private void AssertSingleThreadedMatchesParallel(params string[] words) { var parallel = new Morpher(TraceManager, Language);