diff --git a/CMakeLists.txt b/CMakeLists.txt index f6d4bb2e..35be199b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ endif() # endif() set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) # set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") -project(Thot VERSION 3.5.2 LANGUAGES CXX C) +project(Thot VERSION 3.5.3 LANGUAGES CXX C) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") diff --git a/nuget/Thot.nuspec b/nuget/Thot.nuspec index 0987ba3d..02c01311 100644 --- a/nuget/Thot.nuspec +++ b/nuget/Thot.nuspec @@ -2,7 +2,7 @@ Thot - 3.5.2 + 3.5.3 Thot Daniel Ortiz-Martínez,SIL International sil-lsdev diff --git a/setup.py b/setup.py index 25ec7cc1..34fe6fb8 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ def build_extension(self, ext): # logic and declaration, and simpler if you include description/version in a file. setup( name="sil-thot", - version="3.5.2", + version="3.5.3", author="SIL International", maintainer="Damien Daspit", maintainer_email="damien_daspit@sil.org", diff --git a/src/python_module/module.cc b/src/python_module/module.cc index 268ee873..5a18941b 100644 --- a/src/python_module/module.cc +++ b/src/python_module/module.cc @@ -316,6 +316,7 @@ PYBIND11_MODULE(thot, m) .def(py::init, std::shared_ptr>(), py::arg("direct_model"), py::arg("inverse_model")) .def_property_readonly("num_sentence_pairs", &SymmetrizedAlignmentModel::numSentencePairs) + .def_property_readonly("num_training_alignments", &SymmetrizedAlignmentModel::numTrainingAlignments) .def( "get_sentence_pair", [](SymmetrizedAlignmentModel& model, unsigned int n) { @@ -381,6 +382,7 @@ PYBIND11_MODULE(thot, m) .def("end_training", &AlignmentModel::endTraining) .def_property("emit_training_alignments", &AlignmentModel::getEmitTrainingAlignments, &AlignmentModel::setEmitTrainingAlignments) + .def_property_readonly("num_training_alignments", &AlignmentModel::numTrainingAlignments) .def( "get_training_alignment", [](AlignmentModel& model, size_t n) { diff --git a/src/shared_library/thot.cc b/src/shared_library/thot.cc index b4da90f3..c1ad7b04 100644 --- a/src/shared_library/thot.cc +++ b/src/shared_library/thot.cc @@ -1015,6 +1015,12 @@ extern "C" return alignmentModel->getEmitTrainingAlignments(); } + unsigned int swAlignModel_getNumTrainingAlignments(void* swAlignModelHandle) + { + auto alignmentModel = static_cast(swAlignModelHandle); + return (unsigned int)alignmentModel->numTrainingAlignments(); + } + void swAlignModel_save(void* swAlignModelHandle, const char* prefFileName) { auto alignmentModel = static_cast(swAlignModelHandle); diff --git a/src/shared_library/thot.h b/src/shared_library/thot.h index 1f377ec2..46c4a097 100644 --- a/src/shared_library/thot.h +++ b/src/shared_library/thot.h @@ -231,6 +231,8 @@ extern "C" THOT_API bool swAlignModel_getEmitTrainingAlignments(void* swAlignModelHandle); + THOT_API unsigned int swAlignModel_getNumTrainingAlignments(void* swAlignModelHandle); + THOT_API void swAlignModel_save(void* swAlignModelHandle, const char* prefFileName); THOT_API double swAlignModel_getTranslationProbability(void* swAlignModelHandle, const char* srcWord, diff --git a/src/sw_models/AlignmentModel.h b/src/sw_models/AlignmentModel.h index 4ef1068b..f85c7cb4 100644 --- a/src/sw_models/AlignmentModel.h +++ b/src/sw_models/AlignmentModel.h @@ -54,20 +54,14 @@ class AlignmentModel : public virtual Aligner virtual void train(int verbosity = 0) = 0; virtual void endTraining() = 0; - // Training-alignment functions. When enabled before training, the model - // records the alignment it inferred for every training pair (the same - // per-target form getBestAlignment returns: one entry per target token, the - // 1-based source position it aligns to, 0 = NULL). These are persisted by - // print()/restored by load() as 0-based Pharaoh links in ".aligns". + // Training alignment (transductive) functions + // keeps each training pair's inferred alignment, persisted in ".aligns" virtual void setEmitTrainingAlignments(bool value) = 0; virtual bool getEmitTrainingAlignments() = 0; - // The alignment for a single training pair, mirroring getBestAlignment: fills - // 'alignment' (per target token, 1-based source position, 0 = NULL) and returns - // its log-probability under the model. n is the index in the order pairs were - // added that passed the length filter (the same line index as the ".aligns" - // file). If n is out of range, clears 'alignment' and returns SMALL_LG_NUM. + // bound for getTrainingAlignment; counts filtered pairs, unlike numSentencePairs() + virtual size_t numTrainingAlignments() = 0; + // returns the stored alignment for pair n, indexed as getSentencePair virtual LgProb getTrainingAlignment(size_t n, std::vector& alignment) = 0; - // As above, but fills a WordAlignmentMatrix instead of a per-target vector. virtual LgProb getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) = 0; virtual std::pair loglikelihoodForPairRange(std::pair sentPairRange, int verbosity = 0) = 0; diff --git a/src/sw_models/AlignmentModelBase.cc b/src/sw_models/AlignmentModelBase.cc index e7bff866..0ea723ac 100644 --- a/src/sw_models/AlignmentModelBase.cc +++ b/src/sw_models/AlignmentModelBase.cc @@ -361,6 +361,9 @@ void AlignmentModelBase::loadConfig(const YAML::Node& config) { variationalBayes = config["variationalBayes"].as(); alpha = config["alpha"].as(); + // optional: an unguarded read of a config predating this key throws + if (config["emitTrainingAlignments"]) + emitTrainingAlignments = config["emitTrainingAlignments"].as(); } bool AlignmentModelBase::loadOldConfig(const char* prefFileName, int verbose) @@ -375,6 +378,7 @@ void AlignmentModelBase::createConfig(YAML::Emitter& out) out << YAML::Key << "model" << YAML::Value << getModelTypeStr(); out << YAML::Key << "variationalBayes" << YAML::Value << variationalBayes; out << YAML::Key << "alpha" << YAML::Value << alpha; + out << YAML::Key << "emitTrainingAlignments" << YAML::Value << emitTrainingAlignments; } vector AlignmentModelBase::addNullWordToWidxVec(const vector& vw) @@ -480,9 +484,7 @@ bool AlignmentModelBase::load(const char* prefFileName, int verbose) wordClasses->load(prefFileName, verbose); - // Restore persisted training alignments if present (optional: models trained - // without emitTrainingAlignments have none). Done after the sentence pairs are - // loaded above, since their target lengths are needed to rebuild the form. + // after the sentence pairs: their target lengths rebuild the per-target form loadTrainingAlignments(prefFileName); return THOT_OK; @@ -591,10 +593,13 @@ bool AlignmentModelBase::getEmitTrainingAlignments() return emitTrainingAlignments; } +size_t AlignmentModelBase::numTrainingAlignments() +{ + return trainingAlignments.size(); +} + LgProb AlignmentModelBase::getTrainingAlignment(size_t n, vector& alignment) { - // n is the sentence-handler pair index (the same index getSentencePair takes). - // Out of range: there is no such pair, so return an empty alignment. if (n >= trainingAlignments.size()) { alignment.clear(); @@ -605,10 +610,7 @@ LgProb AlignmentModelBase::getTrainingAlignment(size_t n, vector& Count c; getSentencePair((unsigned int)n, srcStr, trgStr, c); - // An empty stored alignment means the pair was filtered out of training (too - // long, or no alignments were emitted). Mimic getBestAlignment's handling of - // length-invalid input: an all-NULL alignment of the target length, scored - // SMALL_LG_NUM. + // empty = filtered out of training; mimic getBestAlignment on invalid lengths if (trainingAlignments[n].empty()) { alignment.assign(trgStr.size(), 0); @@ -616,8 +618,6 @@ LgProb AlignmentModelBase::getTrainingAlignment(size_t n, vector& } alignment = trainingAlignments[n]; - // The probability is the alignment's score under the model, computed on demand - // (like getBestAlignment) from the stored alignment and the pair's sentences. vector src = strVectorToSrcIndexVector(srcStr); vector trg = strVectorToTrgIndexVector(trgStr); WordAlignmentMatrix waMatrix; @@ -630,8 +630,6 @@ LgProb AlignmentModelBase::getTrainingAlignment(size_t n, WordAlignmentMatrix& b { vector alignment; LgProb logProb = getTrainingAlignment(n, alignment); - // Source length (without NULL) for the matrix dimension; the per-target vector - // gives the target length. PositionIndex slen = 0; if (!alignment.empty()) { @@ -655,7 +653,6 @@ vector AlignmentModelBase::trainingPairSentenceIndices() getSentencePair(n, srcStr, trgStr, c); vector src = strVectorToSrcIndexVector(srcStr); vector trg = strVectorToTrgIndexVector(trgStr); - // Only pairs that pass the length filter were trained on (and emitted). if (sentenceLengthIsOk(src) && sentenceLengthIsOk(trg)) indices.push_back(n); } @@ -668,9 +665,6 @@ void AlignmentModelBase::computeTrainingAlignments() if (!emitTrainingAlignments) return; - // Indexed by sentence-handler pair index (the same index getSentencePair takes), - // so getTrainingAlignment(n) lines up with getSentencePair(n). Pairs that fail - // the length filter (not trained on) keep an empty alignment. trainingAlignments.resize(numSentencePairs()); for (unsigned int n = 0; n < numSentencePairs(); ++n) { @@ -694,9 +688,7 @@ bool AlignmentModelBase::printTrainingAlignments(const char* prefFileName) ofstream af(fileName); if (!af) return THOT_ERROR; - // One Pharaoh line per sentence-handler pair (so line index == pair index): - // 0-based source-target links, with NULL-aligned targets omitted as usual. A - // pair filtered out of training has an empty alignment and writes a blank line. + // line index == pair index; a filtered pair writes a blank line for (const vector& alig : trainingAlignments) { string line; @@ -721,11 +713,8 @@ bool AlignmentModelBase::loadTrainingAlignments(const char* prefFileName) if (!af) return THOT_OK; // optional file - // Line index == sentence-handler pair index. Pharaoh omits NULL-aligned targets, - // so the per-target length comes from the pair's target sentence. A pair that - // fails the length filter was not trained on, so its alignment stays empty - // (matching computeTrainingAlignments); this also disambiguates a blank line for - // a filtered pair from a blank line for a trained pair whose targets are all NULL. + // Pharaoh omits NULL-aligned targets, so target length comes from the corpus; a + // filtered pair stays empty, telling its blank line from an all-NULL one string line; while (getline(af, line)) { diff --git a/src/sw_models/AlignmentModelBase.h b/src/sw_models/AlignmentModelBase.h index 3a92b698..aaf84879 100644 --- a/src/sw_models/AlignmentModelBase.h +++ b/src/sw_models/AlignmentModelBase.h @@ -123,12 +123,10 @@ class AlignmentModelBase : public virtual AlignmentModel // normalizing its chains) override and chain to this for the shared tail. void endTraining() override; - // Training-alignment functions (see AlignmentModel for semantics). The flag - // and storage live here so every model gains the capability; the default - // computeTrainingAlignments() produces them via getBestAlignment, and models - // with a cheaper/better source (e.g. Eflomal's warm argmax) override it. + // Training alignment functions (see AlignmentModel) void setEmitTrainingAlignments(bool value) override; bool getEmitTrainingAlignments() override; + size_t numTrainingAlignments() override; LgProb getTrainingAlignment(size_t n, std::vector& alignment) override; LgProb getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) override; @@ -151,21 +149,11 @@ class AlignmentModelBase : public virtual AlignmentModel bool loadVariationalBayes(const std::string& filename); bool sentenceLengthIsOk(const std::vector sentence); - // Fills trainingAlignments (per training pair that passes the length filter, - // in added order) when emitTrainingAlignments is set; a no-op otherwise. The - // default uses getBestAlignment on each pair; called by each model's - // endTraining before its temporary state is cleared. Models override to use a - // training-time alignment instead of recomputing it; an override must index - // trainingAlignments by sentence-handler pair index (empty for filtered pairs). + // called from endTraining; an override must index by pair index, filtered empty virtual void computeTrainingAlignments(); - // The sentence-handler indices of the pairs that pass the length filter, in - // order. Lets a model that works over the filtered corpus map a corpus position - // back to its sentence-handler pair index. + // pair indices that passed the length filter, in corpus order std::vector trainingPairSentenceIndices(); - // Writes/restores trainingAlignments as 0-based Pharaoh links in - // ".aligns". Called from print()/load(); the read reconstructs the - // exact per-target form (including trailing NULL-aligned targets that Pharaoh - // omits) by replaying the length filter to recover each pair's target length. + // ".aligns", one 0-based Pharaoh line per pair bool printTrainingAlignments(const char* prefFileName); bool loadTrainingAlignments(const char* prefFileName); @@ -176,8 +164,7 @@ class AlignmentModelBase : public virtual AlignmentModel virtual void createConfig(YAML::Emitter& out); bool emitTrainingAlignments = false; - // Indexed by sentence-handler pair index (matching getSentencePair); a pair that - // was filtered out of training holds an empty alignment. + // indexed by pair index; a filtered pair holds an empty alignment std::vector> trainingAlignments; PositionIndex maxSentenceLength = 1024; diff --git a/src/sw_models/EflomalAlignmentModel.cc b/src/sw_models/EflomalAlignmentModel.cc index adc84717..c73a6464 100644 --- a/src/sw_models/EflomalAlignmentModel.cc +++ b/src/sw_models/EflomalAlignmentModel.cc @@ -656,23 +656,17 @@ void EflomalAlignmentModel::endTraining() jumpTable = chains[0].jumpTable; fertilityTable = chains[0].fertilityTable; - // Emit training alignments (if enabled) and clear temporary state via the base - // tail, run here while the chains' converged alignments and the corpus are - // still resident (computeTrainingAlignments needs them before they are cleared). + // emits training alignments and clears temp state; needs the chains and corpus AlignmentModelBase::endTraining(); } -// Stored per target token (1-based source, 0 = NULL), the same form -// getBestAlignment returns. void EflomalAlignmentModel::computeTrainingAlignments() { trainingAlignments.clear(); if (!emitTrainingAlignments) return; - // The corpus is the length-filtered subset in added order; map each corpus - // position back to its sentence-handler pair index so trainingAlignments stays - // indexed by pair index (matching getSentencePair), with filtered pairs empty. + // corpusSrc is the filtered subset, so map each position back to its pair index vector sents = trainingPairSentenceIndices(); trainingAlignments.assign(numSentencePairs(), {}); size_t np = corpusSrc.size(); diff --git a/src/sw_models/SymmetrizedAlignmentModel.cc b/src/sw_models/SymmetrizedAlignmentModel.cc index ed491cbc..c1eb5809 100644 --- a/src/sw_models/SymmetrizedAlignmentModel.cc +++ b/src/sw_models/SymmetrizedAlignmentModel.cc @@ -1,5 +1,7 @@ #include "sw_models/SymmetrizedAlignmentModel.h" +#include + using namespace std; SymmetrizedAlignmentModel::SymmetrizedAlignmentModel(shared_ptr directModel, @@ -19,6 +21,12 @@ int SymmetrizedAlignmentModel::getSentencePair(unsigned int n, vector& s return directModel->getSentencePair(n, srcSentStr, trgSentStr, c); } +size_t SymmetrizedAlignmentModel::numTrainingAlignments() +{ + // getTrainingAlignment combines both directions, so the range is the shorter + return std::min(directModel->numTrainingAlignments(), inverseModel->numTrainingAlignments()); +} + LgProb SymmetrizedAlignmentModel::getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) { LgProb logProb = directModel->getTrainingAlignment(n, bestWaMatrix); diff --git a/src/sw_models/SymmetrizedAlignmentModel.h b/src/sw_models/SymmetrizedAlignmentModel.h index 203cf80f..df293ce5 100644 --- a/src/sw_models/SymmetrizedAlignmentModel.h +++ b/src/sw_models/SymmetrizedAlignmentModel.h @@ -6,17 +6,8 @@ #include -// A symmetrized aligner that also supports transductive alignment, i.e. -// getTrainingAlignment. It combines the per-training-pair alignments inferred by -// a direct model (trained on src->trg) and an inverse model (trained on the same -// pairs with trg/src swapped) using a symmetrization heuristic. -// -// Both models must be trained on the same sentence pairs in the same order (the -// inverse with source and target swapped) and with setEmitTrainingAlignments(true), -// so that index n lines up across the two corpora. -// -// Inherits the symmetrized getBestAlignment overloads and the 'heuristic' -// property from SymmetrizedAligner. +// Both models must be trained on the same pairs in the same order (the inverse +// with source and target swapped) so that index n lines up across the two. class SymmetrizedAlignmentModel : public SymmetrizedAligner { public: @@ -30,6 +21,9 @@ class SymmetrizedAlignmentModel : public SymmetrizedAligner int getSentencePair(unsigned int n, std::vector& srcSentStr, std::vector& trgSentStr, Count& c); + // bound for getTrainingAlignment: the shorter of the two directions' + size_t numTrainingAlignments(); + // The symmetrized alignment for training pair n, mirroring getBestAlignment: // combines the direct and inverse models' training alignments under the current // heuristic. Fills 'alignment' (per target token, 1-based source position, diff --git a/tests/sw_models/EflomalAlignmentModelTest.cc b/tests/sw_models/EflomalAlignmentModelTest.cc index 357ec4df..d83b0842 100644 --- a/tests/sw_models/EflomalAlignmentModelTest.cc +++ b/tests/sw_models/EflomalAlignmentModelTest.cc @@ -294,6 +294,59 @@ TEST(EflomalAlignmentModelTest, configNonDefaultRoundTrip) EXPECT_FALSE(loaded.getAutoIterations()); } +TEST(EflomalAlignmentModelTest, configWrittenBeforeEmitFlagWasPersisted) +{ + // "emitTrainingAlignments" postdates the config format, so a model saved by an + // older version has no such key and it must stay optional. Reading it unguarded + // throws InvalidNode, which load() swallows by falling back to the pre-YAML + // config path -- and because that path reports success when ".var_bayes" is + // absent, load() would return THOT_OK having applied only the keys read before + // the throw, silently resetting every later one to its default. + EflomalAlignmentModel model; + model.setSeed(12345u); + model.setP0(0.15); + model.setAutoIterations(false); + model.setIterations(3, 5, 7); + addTrainingData(model); + model.setEmitTrainingAlignments(true); + train(model, 15); + + std::string prefix = "eflomal_legacy_config_test"; + ASSERT_EQ(model.print(prefix.c_str()), THOT_OK); + + // Strip the key to reproduce a config written before it existed. + std::string configFileName = prefix + ".yml"; + std::vector lines; + { + std::ifstream in(configFileName); + ASSERT_TRUE(in.good()); + for (std::string line; std::getline(in, line);) + if (line.find("emitTrainingAlignments") == std::string::npos) + lines.push_back(line); + } + { + std::ofstream out(configFileName); + for (const std::string& line : lines) + out << line << "\n"; + } + + EflomalAlignmentModel loaded; + ASSERT_EQ(loaded.load(prefix.c_str()), THOT_OK); + + // The absent key leaves the flag at its default, but the rest of the config was + // still applied -- these are read after the base call, so they are what a throw + // would have silently dropped. + EXPECT_FALSE(loaded.getEmitTrainingAlignments()); + EXPECT_EQ(loaded.getSeed(), 12345u); + EXPECT_NEAR(loaded.getP0(), 0.15, kEpsilon); + EXPECT_FALSE(loaded.getAutoIterations()); + EXPECT_EQ(loaded.getIbm1Iters(), 3); + EXPECT_EQ(loaded.getHmmIters(), 5); + EXPECT_EQ(loaded.getFertilityIters(), 7); + // The alignments still came back from ".aligns" despite the flag being unset. + EXPECT_EQ(loaded.numTrainingAlignments(), (size_t)loaded.numSentencePairs()); +} + TEST(EflomalAlignmentModelTest, decodeBurnInClampedWhenExceedsIters) { EflomalAlignmentModel model; diff --git a/tests/sw_models/FastAlignModelTest.cc b/tests/sw_models/FastAlignModelTest.cc index 58d895d0..1d5373e3 100644 --- a/tests/sw_models/FastAlignModelTest.cc +++ b/tests/sw_models/FastAlignModelTest.cc @@ -95,8 +95,8 @@ TEST(FastAlignModelTest, trainingAlignments) ASSERT_EQ(model.print(prefix.c_str()), THOT_OK); FastAlignModel loaded; ASSERT_EQ(loaded.load(prefix.c_str()), THOT_OK); - ASSERT_EQ(loaded.numSentencePairs(), model.numSentencePairs()); - for (unsigned int n = 0; n < model.numSentencePairs(); ++n) + ASSERT_EQ(loaded.numTrainingAlignments(), model.numTrainingAlignments()); + for (size_t n = 0; n < model.numTrainingAlignments(); ++n) { std::vector a1, a2; LgProb p1 = model.getTrainingAlignment(n, a1); @@ -106,6 +106,59 @@ TEST(FastAlignModelTest, trainingAlignments) } } +TEST(FastAlignModelTest, numTrainingAlignments) +{ + // numTrainingAlignments reports how many alignments the model holds, which is not + // the same as how it is configured: the flag can be set on a model that has not + // trained yet, and print() only writes ".aligns" when there are alignments. + FastAlignModel configuredOnly; + configuredOnly.setEmitTrainingAlignments(true); + EXPECT_TRUE(configuredOnly.getEmitTrainingAlignments()); + EXPECT_EQ(configuredOnly.numTrainingAlignments(), 0u); + + FastAlignModel without; + addTrainingData(without); + train(without, 2); + EXPECT_EQ(without.numTrainingAlignments(), 0u); + + FastAlignModel model; + addTrainingData(model); + model.setEmitTrainingAlignments(true); + train(model, 2); + EXPECT_EQ(model.numTrainingAlignments(), (size_t)model.numSentencePairs()); + + // It is the bound getTrainingAlignment enforces, and pairs added after training + // are not covered by it -- so it, not numSentencePairs(), is the loop bound. + addSentencePair(model, "ewnay airpay", "new pair"); + EXPECT_EQ(model.numTrainingAlignments(), (size_t)model.numSentencePairs() - 1); + std::vector tail{9}; + EXPECT_EQ((double)model.getTrainingAlignment(model.numTrainingAlignments(), tail), (double)SMALL_LG_NUM); + EXPECT_TRUE(tail.empty()); + + // A reloaded model restores both the alignments and the flag they were emitted + // under, so retraining it emits them again instead of dropping them. + std::string prefix = "fast_align_has_aligns_test"; + ASSERT_EQ(model.print(prefix.c_str()), THOT_OK); + FastAlignModel loaded; + ASSERT_EQ(loaded.load(prefix.c_str()), THOT_OK); + EXPECT_TRUE(loaded.getEmitTrainingAlignments()); + // ".aligns" covers the pairs that were trained on, not the one added afterwards, + // so the shortfall survives the round trip rather than being papered over. + EXPECT_EQ(loaded.numTrainingAlignments(), model.numTrainingAlignments()); + EXPECT_EQ(loaded.numTrainingAlignments(), (size_t)loaded.numSentencePairs() - 1); + // The flag survived the load, so retraining emits again -- now covering every pair. + train(loaded, 2); + EXPECT_EQ(loaded.numTrainingAlignments(), (size_t)loaded.numSentencePairs()); + + // A model trained without the flag writes no ".aligns", so reloading it holds none. + std::string emptyPrefix = "fast_align_no_aligns_test"; + ASSERT_EQ(without.print(emptyPrefix.c_str()), THOT_OK); + FastAlignModel loadedWithout; + ASSERT_EQ(loadedWithout.load(emptyPrefix.c_str()), THOT_OK); + EXPECT_FALSE(loadedWithout.getEmitTrainingAlignments()); + EXPECT_EQ(loadedWithout.numTrainingAlignments(), 0u); +} + TEST(FastAlignModelTest, trainingAlignmentIndexMatchesSentencePair) { // A pair that fails the length filter (here, an empty source) is not trained on, @@ -157,8 +210,8 @@ TEST(FastAlignModelTest, trainingAlignmentIndexMatchesSentencePair) ASSERT_EQ(model.print(prefix.c_str()), THOT_OK); FastAlignModel loaded; ASSERT_EQ(loaded.load(prefix.c_str()), THOT_OK); - ASSERT_EQ(loaded.numSentencePairs(), model.numSentencePairs()); - for (unsigned int n = 0; n < model.numSentencePairs(); ++n) + ASSERT_EQ(loaded.numTrainingAlignments(), model.numTrainingAlignments()); + for (size_t n = 0; n < model.numTrainingAlignments(); ++n) { std::vector a1, a2; model.getTrainingAlignment(n, a1); diff --git a/tests/sw_models/SymmetrizedAlignmentModelTest.cc b/tests/sw_models/SymmetrizedAlignmentModelTest.cc index d87226db..de273127 100644 --- a/tests/sw_models/SymmetrizedAlignmentModelTest.cc +++ b/tests/sw_models/SymmetrizedAlignmentModelTest.cc @@ -59,6 +59,28 @@ TEST(SymmetrizedAlignmentModelTest, sentencePairAccessors) } } +TEST(SymmetrizedAlignmentModelTest, numTrainingAlignments) +{ + shared_ptr direct, inverse; + shared_ptr model = buildSymmetrizedModel(direct, inverse); + EXPECT_EQ(model->numTrainingAlignments(), direct->numTrainingAlignments()); + EXPECT_EQ(model->numTrainingAlignments(), (size_t)model->numSentencePairs()); + + // getTrainingAlignment combines both directions, so alignments in only one of + // them are not usable. + shared_ptr inverseWithout = make_shared(); + for (unsigned int n = 0; n < direct->numSentencePairs(); ++n) + { + vector src, trg; + Count c; + direct->getSentencePair(n, src, trg, c); + inverseWithout->addSentencePair(trg, src, c); // swapped + } + train(*inverseWithout, 2); + SymmetrizedAlignmentModel halfModel{direct, inverseWithout}; + EXPECT_EQ(halfModel.numTrainingAlignments(), 0u); +} + TEST(SymmetrizedAlignmentModelTest, heuristicNoneReturnsDirect) { shared_ptr direct, inverse;