Filter final templates in analysis

src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs

@@ -42,6 +42,25 @@ public IEnumerable<Word> Apply(Word input)
                 _morpher.TraceManager.BeginUnapplyTemplate(_template, input);
 
             Word inWord = input.Clone();
+            // Do not allow a final template to unapply if the grammar is not partial
+            // and a non-template was last unapplied.
+            if (
+                (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates)
+                && inWord.FinalTemplateState == FinalTemplateState.NonTemplate
+                && _template.IsFinal
+            )
+            {
+                if (_morpher.TraceManager.IsTracing)
+                {
+                    _morpher.TraceManager.TemplateNotUnapplied(
+                        _template,
+                        input,
+                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate,
+                        null
+                    );
+                }
+                return Enumerable.Empty<Word>();
+            }
             inWord.Freeze();
 
             var output = new HashSet<Word>(FreezableEqualityComparer<Word>.Default);

src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs

@@ -12,9 +12,11 @@ namespace SIL.Machine.Morphology.HermitCrab
     /// re-run whenever an <c>Analysis*.cs</c> rule changes:
     /// <list type="bullet">
     /// <item><see cref="MorphologicalRules.AnalysisAffixProcessRule"/>: Shape (FST pattern match),
-    /// <see cref="Word.SyntacticFeatureStruct"/> (unifiability gate), per-rule unapplication count.</item>
+    /// <see cref="Word.SyntacticFeatureStruct"/> (unifiability gate), per-rule unapplication count
+    /// and FinalTemplateState.</item>
     /// <item><see cref="MorphologicalRules.AnalysisCompoundingRule"/>: adds <see cref="Word.NonHeadCount"/>
-    /// (<c>MaxStemCount</c> gate) -- never the non-heads' own content, only the count.</item>
+    /// (<c>MaxStemCount</c> gate) -- never the non-heads' own content, only the count
+    /// and FinalTemplateState.</item>
     /// <item><see cref="MorphologicalRules.AnalysisRealizationalAffixProcessRule"/>: adds
     /// <see cref="Word.RealizationalFeatureStruct"/>.</item>
     /// </list>
@@ -31,6 +33,7 @@ namespace SIL.Machine.Morphology.HermitCrab
         private readonly FeatureStruct _realizationalFS;
         private readonly int _nonHeadCount;
         private readonly IReadOnlyDictionary<IMorphologicalRule, int> _ruleCounts;
+        private readonly FinalTemplateState _finalTemplateState;
         private readonly int _hashCode;
 
         /// <summary>
@@ -62,6 +65,7 @@ private AnalysisStateKey(Word word)
             _realizationalFS = word.RealizationalFeatureStruct;
             _nonHeadCount = word.NonHeadCount;
             _ruleCounts = word.UnappliedRuleCounts;
+            _finalTemplateState = word.FinalTemplateState;
 
             // See PinAndKey for why the key pins these rather than just reading them.
             _shape.Freeze();
@@ -74,6 +78,7 @@ private AnalysisStateKey(Word word)
             hash = hash * 31 + _syntacticFS.GetFrozenHashCode();
             hash = hash * 31 + _realizationalFS.GetFrozenHashCode();
             hash = hash * 31 + _nonHeadCount;
+            hash = hash * 31 + _finalTemplateState.GetHashCode();
             if (_ruleCounts != null)
             {
                 // XOR rather than the usual *31 rolling combine: the multiset is unordered, so entries
@@ -100,6 +105,8 @@ public bool Equals(AnalysisStateKey other)
                 return false;
             if (!_syntacticFS.ValueEquals(other._syntacticFS) || !_realizationalFS.ValueEquals(other._realizationalFS))
                 return false;
+            if (_finalTemplateState != other._finalTemplateState)
+                return false;
             return RuleCountsEqual(_ruleCounts, other._ruleCounts);
         }
 

src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs

@@ -161,9 +161,17 @@ internal IEnumerable<Word> Apply(Word input, ref int alternativeCount)
                     }
                     shapeWord[shape] = mruleOutWord;
                 }
-                output.Add(mruleOutWord);
+                Word newMruleOutWord = mruleOutWord;
+                if (mruleOutWord.FinalTemplateState != FinalTemplateState.None)
+                {
+                    // Clear FinalTemplateState to allow clitics.
+                    newMruleOutWord = mruleOutWord.Clone();
+                    newMruleOutWord.FinalTemplateState = FinalTemplateState.None;
+                    newMruleOutWord.Freeze();
+                }
+                output.Add(newMruleOutWord);
                 if (_morpher.TraceManager.IsTracing)
-                    _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord);
+                    _morpher.TraceManager.EndUnapplyStratum(_stratum, newMruleOutWord);
             }
             return output;
         }

src/SIL.Machine.Morphology.HermitCrab/FinalTemplateState.cs

@@ -0,0 +1,10 @@
+namespace SIL.Machine.Morphology.HermitCrab
+{
+    enum FinalTemplateState : byte
+    {
+        None,
+        NonTemplate,
+        FinalTemplate,
+        NonFinalTemplate,
+    }
+}

src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs

@@ -44,6 +44,7 @@ public interface ITraceManager
 
         void BeginUnapplyTemplate(AffixTemplate template, Word input);
         void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied);
+        void TemplateNotUnapplied(AffixTemplate template, Word input, FailureReason reason, object failureObj);
 
         void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output);
         void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIndex, Word input);

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -65,6 +65,18 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
             RuleSelector = rule => true;
 
             _morphemes = new ReadOnlyObservableCollection<Morpheme>(morphemes);
+            IsPartial = GetPartialMorphemes().Count() > 0;
+        }
+
+        public IEnumerable<Morpheme> GetPartialMorphemes()
+        {
+            var morphemes = new HashSet<Morpheme>();
+            foreach (Morpheme morpheme in _morphemes)
+            {
+                if (morpheme.IsPartial)
+                    morphemes.Add(morpheme);
+            }
+            return morphemes;
         }
 
         public ITraceManager TraceManager
@@ -88,6 +100,16 @@ public ITraceManager TraceManager
         /// </summary>
         public bool MergeEquivalentAnalyses { get; set; }
 
+        /// <summary>
+        /// A Morpher is partial if any of the elements are partial.
+        /// </summary>
+        public bool IsPartial { get; set; }
+
+        /// <summary>
+        /// Enforce final templates even if some of the morphemes were partial.
+        /// </summary>
+        public bool AlwaysEnforceFinalTemplates { get; set; }
+
         /// <summary>
         /// Caps the concurrency used within a single parse or generation -- analysis cascade,
         /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -59,6 +59,12 @@ public IEnumerable<Word> Apply(Word input)
                         outWord.SyntacticFeatureStruct.Add(_rule.RequiredSyntacticFeatureStruct);
                     else if (_rule.OutSyntacticFeatureStruct.IsEmpty)
                         outWord.SyntacticFeatureStruct.Clear();
+                    if (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates)
+                    {
+                        outWord.FinalTemplateState = !_rule.IsTemplateRule
+                            ? FinalTemplateState.NonTemplate
+                            : FinalTemplateState.None;
+                    }
                     outWord.MorphologicalRuleUnapplied(_rule);
                     outWord.Freeze();
                     if (_morpher.TraceManager.IsTracing)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs

@@ -130,6 +130,7 @@ RootAllomorph allo in _morpher.SearchRootAllomorphs(_rule.Stratum, outWord.Curre
                         outWord.SyntacticFeatureStruct.Add(_rule.HeadRequiredSyntacticFeatureStruct);
                     else if (_rule.OutSyntacticFeatureStruct.IsEmpty)
                         outWord.SyntacticFeatureStruct.Clear();
+                    outWord.FinalTemplateState = FinalTemplateState.NonTemplate;
                     outWord.MorphologicalRuleUnapplied(_rule);
 
                     outWord.Freeze();

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs

@@ -63,8 +63,7 @@ public IEnumerable<Word> Apply(Word input)
             if (
                 !_rule.IsTemplateRule
                 && (input.IsLastAppliedRuleFinal ?? false)
-                && !input.IsPartial
-                && !_rule.IsPartial
+                && ((!input.IsPartial && !_rule.IsPartial) || _morpher.AlwaysEnforceFinalTemplates)
             )
             {
                 if (_morpher.TraceManager.IsTracing)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs

@@ -62,7 +62,7 @@ public IEnumerable<Word> Apply(Word input)
                 return Enumerable.Empty<Word>();
             }
 
-            if ((input.IsLastAppliedRuleFinal ?? false) && !input.IsPartial)
+            if ((input.IsLastAppliedRuleFinal ?? false) && (!input.IsPartial || _morpher.AlwaysEnforceFinalTemplates))
             {
                 if (_morpher.TraceManager.IsTracing)
                 {

src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs

@@ -84,6 +84,13 @@ public void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIn
             );
         }
 
+        public void TemplateNotUnapplied(AffixTemplate template, Word input, FailureReason reason, object failureObj)
+        {
+            ((Trace)input.CurrentTrace).Children.Add(
+                new Trace(TraceType.TemplateAnalysisInput, template) { Input = input, FailureReason = reason }
+            );
+        }
+
         public void CompoundingRuleNotUnapplied(
             IMorphologicalRule rule,
             int subruleIndex,

src/SIL.Machine.Morphology.HermitCrab/Word.cs

@@ -28,6 +28,7 @@ public class Word : Freezable<Word>, IAnnotatedData<ShapeNode>, ICloneable<Word>
         private FeatureStruct _realizationalFS;
         private Stratum _stratum;
         private bool? _isLastAppliedRuleFinal;
+        private FinalTemplateState _finalTemplateState;
         private bool _isPartial;
         private readonly Dictionary<string, HashSet<int>> _disjunctiveAllomorphIndices;
         private int _mruleAppCount = 0;
@@ -47,6 +48,7 @@ public Word(RootAllomorph rootAllomorph, FeatureStruct realizationalFS)
             _nonHeadApps = new List<Word>();
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>();
             _isLastAppliedRuleFinal = null;
+            _finalTemplateState = FinalTemplateState.None;
             _disjunctiveAllomorphIndices = new Dictionary<string, HashSet<int>>();
         }
 
@@ -65,6 +67,7 @@ public Word(Stratum stratum, Shape shape)
             _nonHeadApps = new List<Word>();
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>();
             _isLastAppliedRuleFinal = null;
+            _finalTemplateState = FinalTemplateState.None;
             _isPartial = false;
             _disjunctiveAllomorphIndices = new Dictionary<string, HashSet<int>>();
         }
@@ -93,6 +96,7 @@ private Word(Word word, bool cloneNonHeadApps)
             _nonHeadAppIndex = word._nonHeadAppIndex;
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>(word._obligatorySyntacticFeatures);
             _isLastAppliedRuleFinal = word._isLastAppliedRuleFinal;
+            _finalTemplateState = word._finalTemplateState;
             _isPartial = word._isPartial;
             CurrentTrace = word.CurrentTrace;
             AnalysisScope = word.AnalysisScope;
@@ -392,6 +396,16 @@ internal bool? IsLastAppliedRuleFinal
             }
         }
 
+        internal FinalTemplateState FinalTemplateState
+        {
+            get { return _finalTemplateState; }
+            set
+            {
+                CheckFrozen();
+                _finalTemplateState = value;
+            }
+        }
+
         /// <summary>
         /// Gets the number of times the specified morphological rule has been applied.
         /// </summary>
@@ -621,6 +635,7 @@ protected override int FreezeImpl()
             code = code * 31 + _mruleApps.GetSequenceHashCode();
             code = code * 31 + _mruleAppIndex.GetHashCode();
             code = code * 31 + _isLastAppliedRuleFinal.GetHashCode();
+            code = code * 31 + _finalTemplateState.GetHashCode();
             return code;
         }
 
@@ -640,7 +655,8 @@ public override bool ValueEquals(Word other)
                 && _rootAllomorph == other._rootAllomorph
                 && _mruleApps.SequenceEqual(other._mruleApps)
                 && _mruleAppIndex == other._mruleAppIndex
-                && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal;
+                && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal
+                && _finalTemplateState == other._finalTemplateState;
         }
 
         public Word Clone()