Add numTrainingAlignments and persist emitTrainingAlignments

CMakeLists.txt

@@ -8,7 +8,7 @@ endif()
 # endif()
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 # set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
-project(Thot VERSION 3.5.2 LANGUAGES CXX C)
+project(Thot VERSION 3.5.3 LANGUAGES CXX C)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 

nuget/Thot.nuspec

@@ -2,7 +2,7 @@
 <package>
 	<metadata>
 		<id>Thot</id>
-		<version>3.5.2</version>
+		<version>3.5.3</version>
 		<title>Thot</title>
 		<authors>Daniel Ortiz-Martínez,SIL International</authors>
 		<owners>sil-lsdev</owners>

setup.py

@@ -143,7 +143,7 @@ def build_extension(self, ext):
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="sil-thot",
-    version="3.5.2",
+    version="3.5.3",
     author="SIL International",
     maintainer="Damien Daspit",
     maintainer_email="damien_daspit@sil.org",

src/python_module/module.cc

@@ -316,6 +316,7 @@ PYBIND11_MODULE(thot, m)
       .def(py::init<std::shared_ptr<AlignmentModel>, std::shared_ptr<AlignmentModel>>(), py::arg("direct_model"),
            py::arg("inverse_model"))
       .def_property_readonly("num_sentence_pairs", &SymmetrizedAlignmentModel::numSentencePairs)
+      .def_property_readonly("num_training_alignments", &SymmetrizedAlignmentModel::numTrainingAlignments)
       .def(
           "get_sentence_pair",
           [](SymmetrizedAlignmentModel& model, unsigned int n) {
@@ -381,6 +382,7 @@ PYBIND11_MODULE(thot, m)
       .def("end_training", &AlignmentModel::endTraining)
       .def_property("emit_training_alignments", &AlignmentModel::getEmitTrainingAlignments,
                     &AlignmentModel::setEmitTrainingAlignments)
+      .def_property_readonly("num_training_alignments", &AlignmentModel::numTrainingAlignments)
       .def(
           "get_training_alignment",
           [](AlignmentModel& model, size_t n) {

src/shared_library/thot.cc

@@ -1015,6 +1015,12 @@ extern "C"
     return alignmentModel->getEmitTrainingAlignments();
   }
 
+  unsigned int swAlignModel_getNumTrainingAlignments(void* swAlignModelHandle)
+  {
+    auto alignmentModel = static_cast<AlignmentModel*>(swAlignModelHandle);
+    return (unsigned int)alignmentModel->numTrainingAlignments();
+  }
+
   void swAlignModel_save(void* swAlignModelHandle, const char* prefFileName)
   {
     auto alignmentModel = static_cast<AlignmentModel*>(swAlignModelHandle);

src/shared_library/thot.h

@@ -231,6 +231,8 @@ extern "C"
 
   THOT_API bool swAlignModel_getEmitTrainingAlignments(void* swAlignModelHandle);
 
+  THOT_API unsigned int swAlignModel_getNumTrainingAlignments(void* swAlignModelHandle);
+
   THOT_API void swAlignModel_save(void* swAlignModelHandle, const char* prefFileName);
 
   THOT_API double swAlignModel_getTranslationProbability(void* swAlignModelHandle, const char* srcWord,

src/sw_models/AlignmentModel.h

@@ -54,20 +54,14 @@ class AlignmentModel : public virtual Aligner
   virtual void train(int verbosity = 0) = 0;
   virtual void endTraining() = 0;
 
-  // Training-alignment functions. When enabled before training, the model
-  // records the alignment it inferred for every training pair (the same
-  // per-target form getBestAlignment returns: one entry per target token, the
-  // 1-based source position it aligns to, 0 = NULL). These are persisted by
-  // print()/restored by load() as 0-based Pharaoh links in "<prefix>.aligns".
+  // Training alignment (transductive) functions
+  // keeps each training pair's inferred alignment, persisted in "<prefix>.aligns"
   virtual void setEmitTrainingAlignments(bool value) = 0;
   virtual bool getEmitTrainingAlignments() = 0;
-  // The alignment for a single training pair, mirroring getBestAlignment: fills
-  // 'alignment' (per target token, 1-based source position, 0 = NULL) and returns
-  // its log-probability under the model. n is the index in the order pairs were
-  // added that passed the length filter (the same line index as the ".aligns"
-  // file). If n is out of range, clears 'alignment' and returns SMALL_LG_NUM.
+  // bound for getTrainingAlignment; counts filtered pairs, unlike numSentencePairs()
+  virtual size_t numTrainingAlignments() = 0;
+  // returns the stored alignment for pair n, indexed as getSentencePair
   virtual LgProb getTrainingAlignment(size_t n, std::vector<PositionIndex>& alignment) = 0;
-  // As above, but fills a WordAlignmentMatrix instead of a per-target vector.
   virtual LgProb getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) = 0;
   virtual std::pair<double, double> loglikelihoodForPairRange(std::pair<unsigned int, unsigned int> sentPairRange,
                                                               int verbosity = 0) = 0;

src/sw_models/AlignmentModelBase.cc

@@ -361,6 +361,9 @@ void AlignmentModelBase::loadConfig(const YAML::Node& config)
 {
   variationalBayes = config["variationalBayes"].as<bool>();
   alpha = config["alpha"].as<double>();
+  // optional: an unguarded read of a config predating this key throws
+  if (config["emitTrainingAlignments"])
+    emitTrainingAlignments = config["emitTrainingAlignments"].as<bool>();
 }
 
 bool AlignmentModelBase::loadOldConfig(const char* prefFileName, int verbose)
@@ -375,6 +378,7 @@ void AlignmentModelBase::createConfig(YAML::Emitter& out)
   out << YAML::Key << "model" << YAML::Value << getModelTypeStr();
   out << YAML::Key << "variationalBayes" << YAML::Value << variationalBayes;
   out << YAML::Key << "alpha" << YAML::Value << alpha;
+  out << YAML::Key << "emitTrainingAlignments" << YAML::Value << emitTrainingAlignments;
 }
 
 vector<WordIndex> AlignmentModelBase::addNullWordToWidxVec(const vector<WordIndex>& vw)
@@ -480,9 +484,7 @@ bool AlignmentModelBase::load(const char* prefFileName, int verbose)
 
     wordClasses->load(prefFileName, verbose);
 
-    // Restore persisted training alignments if present (optional: models trained
-    // without emitTrainingAlignments have none). Done after the sentence pairs are
-    // loaded above, since their target lengths are needed to rebuild the form.
+    // after the sentence pairs: their target lengths rebuild the per-target form
     loadTrainingAlignments(prefFileName);
 
     return THOT_OK;
@@ -591,10 +593,13 @@ bool AlignmentModelBase::getEmitTrainingAlignments()
   return emitTrainingAlignments;
 }
 
+size_t AlignmentModelBase::numTrainingAlignments()
+{
+  return trainingAlignments.size();
+}
+
 LgProb AlignmentModelBase::getTrainingAlignment(size_t n, vector<PositionIndex>& alignment)
 {
-  // n is the sentence-handler pair index (the same index getSentencePair takes).
-  // Out of range: there is no such pair, so return an empty alignment.
   if (n >= trainingAlignments.size())
   {
     alignment.clear();
@@ -605,19 +610,14 @@ LgProb AlignmentModelBase::getTrainingAlignment(size_t n, vector<PositionIndex>&
   Count c;
   getSentencePair((unsigned int)n, srcStr, trgStr, c);
 
-  // An empty stored alignment means the pair was filtered out of training (too
-  // long, or no alignments were emitted). Mimic getBestAlignment's handling of
-  // length-invalid input: an all-NULL alignment of the target length, scored
-  // SMALL_LG_NUM.
+  // empty = filtered out of training; mimic getBestAlignment on invalid lengths
   if (trainingAlignments[n].empty())
   {
     alignment.assign(trgStr.size(), 0);
     return SMALL_LG_NUM;
   }
   alignment = trainingAlignments[n];
 
-  // The probability is the alignment's score under the model, computed on demand
-  // (like getBestAlignment) from the stored alignment and the pair's sentences.
   vector<WordIndex> src = strVectorToSrcIndexVector(srcStr);
   vector<WordIndex> trg = strVectorToTrgIndexVector(trgStr);
   WordAlignmentMatrix waMatrix;
@@ -630,8 +630,6 @@ LgProb AlignmentModelBase::getTrainingAlignment(size_t n, WordAlignmentMatrix& b
 {
   vector<PositionIndex> alignment;
   LgProb logProb = getTrainingAlignment(n, alignment);
-  // Source length (without NULL) for the matrix dimension; the per-target vector
-  // gives the target length.
   PositionIndex slen = 0;
   if (!alignment.empty())
   {
@@ -655,7 +653,6 @@ vector<unsigned int> AlignmentModelBase::trainingPairSentenceIndices()
     getSentencePair(n, srcStr, trgStr, c);
     vector<WordIndex> src = strVectorToSrcIndexVector(srcStr);
     vector<WordIndex> trg = strVectorToTrgIndexVector(trgStr);
-    // Only pairs that pass the length filter were trained on (and emitted).
     if (sentenceLengthIsOk(src) && sentenceLengthIsOk(trg))
       indices.push_back(n);
   }
@@ -668,9 +665,6 @@ void AlignmentModelBase::computeTrainingAlignments()
   if (!emitTrainingAlignments)
     return;
 
-  // Indexed by sentence-handler pair index (the same index getSentencePair takes),
-  // so getTrainingAlignment(n) lines up with getSentencePair(n). Pairs that fail
-  // the length filter (not trained on) keep an empty alignment.
   trainingAlignments.resize(numSentencePairs());
   for (unsigned int n = 0; n < numSentencePairs(); ++n)
   {
@@ -694,9 +688,7 @@ bool AlignmentModelBase::printTrainingAlignments(const char* prefFileName)
   ofstream af(fileName);
   if (!af)
     return THOT_ERROR;
-  // One Pharaoh line per sentence-handler pair (so line index == pair index):
-  // 0-based source-target links, with NULL-aligned targets omitted as usual. A
-  // pair filtered out of training has an empty alignment and writes a blank line.
+  // line index == pair index; a filtered pair writes a blank line
   for (const vector<PositionIndex>& alig : trainingAlignments)
   {
     string line;
@@ -721,11 +713,8 @@ bool AlignmentModelBase::loadTrainingAlignments(const char* prefFileName)
   if (!af)
     return THOT_OK; // optional file
 
-  // Line index == sentence-handler pair index. Pharaoh omits NULL-aligned targets,
-  // so the per-target length comes from the pair's target sentence. A pair that
-  // fails the length filter was not trained on, so its alignment stays empty
-  // (matching computeTrainingAlignments); this also disambiguates a blank line for
-  // a filtered pair from a blank line for a trained pair whose targets are all NULL.
+  // Pharaoh omits NULL-aligned targets, so target length comes from the corpus; a
+  // filtered pair stays empty, telling its blank line from an all-NULL one
   string line;
   while (getline(af, line))
   {

src/sw_models/AlignmentModelBase.h

@@ -123,12 +123,10 @@ class AlignmentModelBase : public virtual AlignmentModel
   // normalizing its chains) override and chain to this for the shared tail.
   void endTraining() override;
 
-  // Training-alignment functions (see AlignmentModel for semantics). The flag
-  // and storage live here so every model gains the capability; the default
-  // computeTrainingAlignments() produces them via getBestAlignment, and models
-  // with a cheaper/better source (e.g. Eflomal's warm argmax) override it.
+  // Training alignment functions (see AlignmentModel)
   void setEmitTrainingAlignments(bool value) override;
   bool getEmitTrainingAlignments() override;
+  size_t numTrainingAlignments() override;
   LgProb getTrainingAlignment(size_t n, std::vector<PositionIndex>& alignment) override;
   LgProb getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) override;
 
@@ -151,21 +149,11 @@ class AlignmentModelBase : public virtual AlignmentModel
   bool loadVariationalBayes(const std::string& filename);
   bool sentenceLengthIsOk(const std::vector<WordIndex> sentence);
 
-  // Fills trainingAlignments (per training pair that passes the length filter,
-  // in added order) when emitTrainingAlignments is set; a no-op otherwise. The
-  // default uses getBestAlignment on each pair; called by each model's
-  // endTraining before its temporary state is cleared. Models override to use a
-  // training-time alignment instead of recomputing it; an override must index
-  // trainingAlignments by sentence-handler pair index (empty for filtered pairs).
+  // called from endTraining; an override must index by pair index, filtered empty
   virtual void computeTrainingAlignments();
-  // The sentence-handler indices of the pairs that pass the length filter, in
-  // order. Lets a model that works over the filtered corpus map a corpus position
-  // back to its sentence-handler pair index.
+  // pair indices that passed the length filter, in corpus order
   std::vector<unsigned int> trainingPairSentenceIndices();
-  // Writes/restores trainingAlignments as 0-based Pharaoh links in
-  // "<prefix>.aligns". Called from print()/load(); the read reconstructs the
-  // exact per-target form (including trailing NULL-aligned targets that Pharaoh
-  // omits) by replaying the length filter to recover each pair's target length.
+  // "<prefix>.aligns", one 0-based Pharaoh line per pair
   bool printTrainingAlignments(const char* prefFileName);
   bool loadTrainingAlignments(const char* prefFileName);
 
@@ -176,8 +164,7 @@ class AlignmentModelBase : public virtual AlignmentModel
   virtual void createConfig(YAML::Emitter& out);
 
   bool emitTrainingAlignments = false;
-  // Indexed by sentence-handler pair index (matching getSentencePair); a pair that
-  // was filtered out of training holds an empty alignment.
+  // indexed by pair index; a filtered pair holds an empty alignment
   std::vector<std::vector<PositionIndex>> trainingAlignments;
 
   PositionIndex maxSentenceLength = 1024;

src/sw_models/EflomalAlignmentModel.cc

@@ -656,23 +656,17 @@ void EflomalAlignmentModel::endTraining()
   jumpTable = chains[0].jumpTable;
   fertilityTable = chains[0].fertilityTable;
 
-  // Emit training alignments (if enabled) and clear temporary state via the base
-  // tail, run here while the chains' converged alignments and the corpus are
-  // still resident (computeTrainingAlignments needs them before they are cleared).
+  // emits training alignments and clears temp state; needs the chains and corpus
   AlignmentModelBase::endTraining();
 }
 
-// Stored per target token (1-based source, 0 = NULL), the same form
-// getBestAlignment returns.
 void EflomalAlignmentModel::computeTrainingAlignments()
 {
   trainingAlignments.clear();
   if (!emitTrainingAlignments)
     return;
 
-  // The corpus is the length-filtered subset in added order; map each corpus
-  // position back to its sentence-handler pair index so trainingAlignments stays
-  // indexed by pair index (matching getSentencePair), with filtered pairs empty.
+  // corpusSrc is the filtered subset, so map each position back to its pair index
   vector<unsigned int> sents = trainingPairSentenceIndices();
   trainingAlignments.assign(numSentencePairs(), {});
   size_t np = corpusSrc.size();

src/sw_models/SymmetrizedAlignmentModel.cc

@@ -1,5 +1,7 @@
 #include "sw_models/SymmetrizedAlignmentModel.h"
 
+#include <algorithm>
+
 using namespace std;
 
 SymmetrizedAlignmentModel::SymmetrizedAlignmentModel(shared_ptr<AlignmentModel> directModel,
@@ -19,6 +21,12 @@ int SymmetrizedAlignmentModel::getSentencePair(unsigned int n, vector<string>& s
   return directModel->getSentencePair(n, srcSentStr, trgSentStr, c);
 }
 
+size_t SymmetrizedAlignmentModel::numTrainingAlignments()
+{
+  // getTrainingAlignment combines both directions, so the range is the shorter
+  return std::min(directModel->numTrainingAlignments(), inverseModel->numTrainingAlignments());
+}
+
 LgProb SymmetrizedAlignmentModel::getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix)
 {
   LgProb logProb = directModel->getTrainingAlignment(n, bestWaMatrix);

src/sw_models/SymmetrizedAlignmentModel.h

@@ -6,17 +6,8 @@
 
 #include <memory>
 
-// A symmetrized aligner that also supports transductive alignment, i.e.
-// getTrainingAlignment. It combines the per-training-pair alignments inferred by
-// a direct model (trained on src->trg) and an inverse model (trained on the same
-// pairs with trg/src swapped) using a symmetrization heuristic.
-//
-// Both models must be trained on the same sentence pairs in the same order (the
-// inverse with source and target swapped) and with setEmitTrainingAlignments(true),
-// so that index n lines up across the two corpora.
-//
-// Inherits the symmetrized getBestAlignment overloads and the 'heuristic'
-// property from SymmetrizedAligner.
+// Both models must be trained on the same pairs in the same order (the inverse
+// with source and target swapped) so that index n lines up across the two.
 class SymmetrizedAlignmentModel : public SymmetrizedAligner
 {
 public:
@@ -30,6 +21,9 @@ class SymmetrizedAlignmentModel : public SymmetrizedAligner
   int getSentencePair(unsigned int n, std::vector<std::string>& srcSentStr, std::vector<std::string>& trgSentStr,
                       Count& c);
 
+  // bound for getTrainingAlignment: the shorter of the two directions'
+  size_t numTrainingAlignments();
+
   // The symmetrized alignment for training pair n, mirroring getBestAlignment:
   // combines the direct and inverse models' training alignments under the current
   // heuristic. Fills 'alignment' (per target token, 1-based source position,

tests/sw_models/EflomalAlignmentModelTest.cc

@@ -294,6 +294,59 @@ TEST(EflomalAlignmentModelTest, configNonDefaultRoundTrip)
   EXPECT_FALSE(loaded.getAutoIterations());
 }
 
+TEST(EflomalAlignmentModelTest, configWrittenBeforeEmitFlagWasPersisted)
+{
+  // "emitTrainingAlignments" postdates the config format, so a model saved by an
+  // older version has no such key and it must stay optional. Reading it unguarded
+  // throws InvalidNode, which load() swallows by falling back to the pre-YAML
+  // config path -- and because that path reports success when ".var_bayes" is
+  // absent, load() would return THOT_OK having applied only the keys read before
+  // the throw, silently resetting every later one to its default.
+  EflomalAlignmentModel model;
+  model.setSeed(12345u);
+  model.setP0(0.15);
+  model.setAutoIterations(false);
+  model.setIterations(3, 5, 7);
+  addTrainingData(model);
+  model.setEmitTrainingAlignments(true);
+  train(model, 15);
+
+  std::string prefix = "eflomal_legacy_config_test";
+  ASSERT_EQ(model.print(prefix.c_str()), THOT_OK);
+
+  // Strip the key to reproduce a config written before it existed.
+  std::string configFileName = prefix + ".yml";
+  std::vector<std::string> lines;
+  {
+    std::ifstream in(configFileName);
+    ASSERT_TRUE(in.good());
+    for (std::string line; std::getline(in, line);)
+      if (line.find("emitTrainingAlignments") == std::string::npos)
+        lines.push_back(line);
+  }
+  {
+    std::ofstream out(configFileName);
+    for (const std::string& line : lines)
+      out << line << "\n";
+  }
+
+  EflomalAlignmentModel loaded;
+  ASSERT_EQ(loaded.load(prefix.c_str()), THOT_OK);
+
+  // The absent key leaves the flag at its default, but the rest of the config was
+  // still applied -- these are read after the base call, so they are what a throw
+  // would have silently dropped.
+  EXPECT_FALSE(loaded.getEmitTrainingAlignments());
+  EXPECT_EQ(loaded.getSeed(), 12345u);
+  EXPECT_NEAR(loaded.getP0(), 0.15, kEpsilon);
+  EXPECT_FALSE(loaded.getAutoIterations());
+  EXPECT_EQ(loaded.getIbm1Iters(), 3);
+  EXPECT_EQ(loaded.getHmmIters(), 5);
+  EXPECT_EQ(loaded.getFertilityIters(), 7);
+  // The alignments still came back from ".aligns" despite the flag being unset.
+  EXPECT_EQ(loaded.numTrainingAlignments(), (size_t)loaded.numSentencePairs());
+}
+
 TEST(EflomalAlignmentModelTest, decodeBurnInClampedWhenExceedsIters)
 {
   EflomalAlignmentModel model;