From 0b17662cd49fdbb0689d2224caa231d531040d24 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 07:42:49 +0100 Subject: [PATCH 01/10] METK-183 MarsLanguage immutable + lazy loading + caching --- src/metkit/mars/MarsExpansion.cc | 32 ++++++--------------- src/metkit/mars/MarsExpansion.h | 6 +--- src/metkit/mars/MarsLanguage.cc | 42 +++++++++++++++++----------- src/metkit/mars/MarsLanguage.h | 11 +++----- src/metkit/mars/Type.cc | 33 +++++++--------------- src/metkit/mars/Type.h | 4 --- src/metkit/mars/TypeEnum.cc | 5 ---- src/metkit/mars/TypeEnum.h | 1 - src/metkit/mars/TypeParam.cc | 5 ---- src/metkit/mars/TypeParam.h | 1 - src/metkit/odb/OdbDecoder.cc | 2 +- src/metkit/odb/OdbMetadataDecoder.cc | 2 +- src/metkit/odb/OdbMetadataDecoder.h | 4 +-- src/metkit/odb/OdbToRequest.cc | 2 +- tests/test_date.cc | 3 +- tests/test_expand.cc | 6 ++-- tests/test_language.cc | 28 +++++++++---------- tests/test_mars_language.cc | 8 +++--- tests/test_mars_language_strict.cc | 9 ++---- tests/test_obstype.cc | 6 ++-- tests/test_param_axis.cc | 5 ++-- tests/test_step.cc | 3 +- tests/test_time.cc | 3 +- tests/test_type_levelist.cc | 3 +- 24 files changed, 85 insertions(+), 139 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index d78f8d553..95f90d63d 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -19,57 +19,41 @@ namespace metkit::mars { MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {} -MarsExpansion::~MarsExpansion() { - for (auto& language : languages_) { - delete language.second; - } -} - -void MarsExpansion::reset() { - for (auto& language : languages_) { - language.second->reset(); - } -} - -MarsLanguage& MarsExpansion::language(const std::string& verb) { +const MarsLanguage& MarsExpansion::language(const std::string& verb) { auto v = MarsLanguage::expandVerb(verb); - - if (auto j = languages_.find(v); j != languages_.end()) { - return *(*j).second; - } - - auto j = languages_.emplace(v, new MarsLanguage(v)).first; - return *(*j).second; + return MarsLanguage::get(v); } - std::vector MarsExpansion::expand(const std::vector& requests) { + MarsRequest ctx; std::vector result; result.reserve(requests.size()); // Implement inheritence for (const auto& request : requests) { auto& lang = language(request.verb()); - result.emplace_back(lang.expand(request, inherit_, strict_)); + result.emplace_back(lang.expand(request, ctx, inherit_, strict_)); } return result; } std::vector MarsExpansion::expand(const std::vector& requests) { + MarsRequest ctx; std::vector result; result.reserve(requests.size()); for (const auto& request : requests) { auto& lang = language(request.verb()); - result.emplace_back(lang.expand(request, inherit_, strict_)); + result.emplace_back(lang.expand(request, ctx, inherit_, strict_)); } return result; } MarsRequest MarsExpansion::expand(const MarsRequest& request) { - return language(request.verb()).expand(request, inherit_, strict_); + MarsRequest ctx; + return language(request.verb()).expand(request, ctx, inherit_, strict_); } void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) { diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index cf14889b8..c8639ee19 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -58,9 +58,6 @@ class MarsExpansion : public eckit::NonCopyable { public: MarsExpansion(bool inherit, bool strict = false); - ~MarsExpansion(); - - void reset(); MarsRequest expand(const MarsRequest&); std::vector expand(const std::vector&); @@ -71,9 +68,8 @@ class MarsExpansion : public eckit::NonCopyable { private: - MarsLanguage& language(const std::string& verb); + const MarsLanguage& language(const std::string& verb); - std::map languages_; bool inherit_; bool strict_; }; diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index ce6f2bc5a..157778f7d 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -231,14 +231,18 @@ MarsLanguage::~MarsLanguage() { } } -eckit::PathName MarsLanguage::languageYamlFile() { - return metkit::LibMetkit::languageYamlFile(); -} - -void MarsLanguage::reset() { - for (auto& [k, t] : types_) { - t->reset(); +const MarsLanguage& MarsLanguage::get(const std::string& verb) { + static std::mutex mutex; + static std::map instances; + + std::lock_guard lock(mutex); + auto it = instances.find(verb); + if (it != instances.end()) { + return *(it->second); } + auto [newIt, inserted] = instances.emplace(verb, new MarsLanguage(verb)); + ASSERT(inserted); + return *(newIt->second); } eckit::Value MarsLanguage::jsonFile(const std::string& name) { @@ -439,7 +443,8 @@ Type* MarsLanguage::type(const std::string& name) const { } -MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) { +// MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) { +MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool inherit, bool strict) const { MarsRequest result(verb_); try { @@ -478,7 +483,7 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict const std::string& s = eckit::StringTools::lower(values[0]); if (s == "off") { result.unsetValues(p); - type(p)->reset(); + ctx.unsetValues(p); continue; } if (s == "all" && type(p)->multiple()) { @@ -496,14 +501,14 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict if (inherit) { for (const auto& [k, t] : typesByAxisOrder_) { if (t != nullptr && result.countValues(k) == 0) { - t->setDefaults(result); + if (ctx.has(k)) { + result.setValuesTyped(t, ctx.values(k)); + } + else { + t->setDefaults(result); + } } } - - result.getParams(params); - for (std::vector::const_iterator k = params.begin(); k != params.end(); ++k) { - type(*k)->setInheritance(result.values(*k)); - } } result.getParams(params); @@ -522,6 +527,9 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict oss << e.what() << " request=" << r << ", expanded=" << result; throw eckit::UserError(oss.str()); } + if (inherit) { + ctx = result; + } return result; } @@ -531,7 +539,7 @@ const std::string& MarsLanguage::verb() const { } void MarsLanguage::flatten(const MarsRequest& request, const std::vector& params, size_t i, - MarsRequest& result, FlattenCallback& callback) { + MarsRequest& result, FlattenCallback& callback) const { if (i == params.size()) { callback(result); return; @@ -553,7 +561,7 @@ void MarsLanguage::flatten(const MarsRequest& request, const std::vector params; request.getParams(params); diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index 7e4310bea..f06f6350b 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -51,15 +51,11 @@ class MarsLanguage : private eckit::NonCopyable { ~MarsLanguage(); - MarsRequest expand(const MarsRequest& r, bool inherit, bool strict); - - void reset(); + MarsRequest expand(const MarsRequest& r, MarsRequest& ctx, bool inherit, bool strict) const; const std::string& verb() const; - void flatten(const MarsRequest& request, FlattenCallback& callback); - - static eckit::PathName languageYamlFile(); + void flatten(const MarsRequest& request, FlattenCallback& callback) const; Type* type(const std::string& name) const; @@ -73,6 +69,7 @@ class MarsLanguage : private eckit::NonCopyable { public: // class methods static std::string expandVerb(const std::string& verb); + static const MarsLanguage& get(const std::string& verb); static std::string bestMatch(const std::string& name, const std::vector& values, bool fail, bool quiet, bool fullMatch, const std::map& aliases = {}); @@ -83,7 +80,7 @@ class MarsLanguage : private eckit::NonCopyable { private: // methods void flatten(const MarsRequest& request, const std::vector& params, size_t i, MarsRequest& result, - FlattenCallback& callback); + FlattenCallback& callback) const; void parseModifier(ModifierType typ, std::shared_ptr ctx, size_t maxIndex, const eckit::Value& mod); private: // members diff --git a/src/metkit/mars/Type.cc b/src/metkit/mars/Type.cc index fe309e801..4e71a8742 100644 --- a/src/metkit/mars/Type.cc +++ b/src/metkit/mars/Type.cc @@ -315,32 +315,23 @@ void Type::expand(std::vector& values, const MarsRequest& request) } void Type::setDefaults(MarsRequest& request) { - if (inheritance_) { - request.setValuesTyped(this, inheritance_.value()); + bool unset = false; + for (const auto& unsetContext : unsets_) { + if (unsetContext->matches(request)) { + unset = true; + break; + } } - else { - bool unset = false; - for (const auto& unsetContext : unsets_) { - if (unsetContext->matches(request)) { - unset = true; + if (!unset) { + for (const auto& [defaultContext, values] : defaults_) { + if (defaultContext->matches(request)) { + patchRequest(request, values); break; } } - if (!unset) { - for (const auto& [defaultContext, values] : defaults_) { - if (defaultContext->matches(request)) { - patchRequest(request, values); - break; - } - } - } } } -void Type::setInheritance(const std::vector& inheritance) { - inheritance_ = inheritance; -} - const std::vector& Type::flattenValues(const MarsRequest& request) { return request.values(name_); } @@ -349,10 +340,6 @@ void Type::clearDefaults() { defaults_.clear(); } -void Type::reset() { - inheritance_.reset(); -} - const std::string& Type::name() const { return name_; } diff --git a/src/metkit/mars/Type.h b/src/metkit/mars/Type.h index d55c59e1c..a0c2d3c91 100644 --- a/src/metkit/mars/Type.h +++ b/src/metkit/mars/Type.h @@ -195,10 +195,8 @@ class Type : public eckit::Counted { std::string tidy(const std::string& value, const MarsRequest& request = {}) const; virtual void setDefaults(MarsRequest& request); - virtual void setInheritance(const std::vector& inheritance); virtual void check(const std::vector& values) const; virtual void clearDefaults(); - virtual void reset(); virtual void pass2(MarsRequest& request); virtual void finalise(MarsRequest& request, bool strict); @@ -245,8 +243,6 @@ class Type : public eckit::Counted { std::map, std::vector> sets_; std::set> unsets_; - std::optional> inheritance_; - std::unique_ptr toByList_; std::map&, std::vector&)>> filters_; diff --git a/src/metkit/mars/TypeEnum.cc b/src/metkit/mars/TypeEnum.cc index b48c4a3f2..d01b6a0bf 100644 --- a/src/metkit/mars/TypeEnum.cc +++ b/src/metkit/mars/TypeEnum.cc @@ -138,11 +138,6 @@ std::optional>> TypeEnum:: return std::nullopt; } - -void TypeEnum::reset() { - Type::reset(); -} - static TypeBuilder type("enum"); //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/TypeEnum.h b/src/metkit/mars/TypeEnum.h index 34c890066..0a0f2732d 100644 --- a/src/metkit/mars/TypeEnum.h +++ b/src/metkit/mars/TypeEnum.h @@ -38,7 +38,6 @@ class TypeEnum : public Type { const std::string& value) const override; void print(std::ostream& out) const override; - void reset() override; bool expand(std::string& value, const MarsRequest& request) const override; std::map::const_iterator find(const std::string& value) const; diff --git a/src/metkit/mars/TypeParam.cc b/src/metkit/mars/TypeParam.cc index 80175b3f1..8e8ace359 100644 --- a/src/metkit/mars/TypeParam.cc +++ b/src/metkit/mars/TypeParam.cc @@ -511,11 +511,6 @@ bool TypeParam::expand(std::string&, const MarsRequest&) const { return true; } -void TypeParam::reset() { - // cache_.clear(); - Type::reset(); -} - static TypeBuilder type("param"); //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/TypeParam.h b/src/metkit/mars/TypeParam.h index d50fb9e14..195457489 100644 --- a/src/metkit/mars/TypeParam.h +++ b/src/metkit/mars/TypeParam.h @@ -35,7 +35,6 @@ class TypeParam : public Type { bool firstRule_; void print(std::ostream& out) const override; - void reset() override; void pass2(MarsRequest& request) override; bool expand(std::string& value, const MarsRequest& request) const override; }; diff --git a/src/metkit/odb/OdbDecoder.cc b/src/metkit/odb/OdbDecoder.cc index 62eae9fec..f72a94a17 100644 --- a/src/metkit/odb/OdbDecoder.cc +++ b/src/metkit/odb/OdbDecoder.cc @@ -47,7 +47,7 @@ void OdbDecoder::getMetadata(const eckit::message::Message& msg, eckit::message: odc::api::Reader reader(*handle, false); odc::api::Frame frame; - metkit::mars::MarsLanguage language("retrieve"); + const metkit::mars::MarsLanguage& language = mars::MarsLanguage::get("retrieve"); OdbMetadataDecoder setter(gather, options, language); while ((frame = reader.next())) { diff --git a/src/metkit/odb/OdbMetadataDecoder.cc b/src/metkit/odb/OdbMetadataDecoder.cc index fb3428a52..aef237fa5 100644 --- a/src/metkit/odb/OdbMetadataDecoder.cc +++ b/src/metkit/odb/OdbMetadataDecoder.cc @@ -94,7 +94,7 @@ void OdbMetadataDecoder::visit(const std::string& columnName, const std::set& OdbMetadataDecoder::OdbMetadataDecoder(eckit::message::MetadataGatherer& gather, const eckit::message::GetMetadataOptions& options, - metkit::mars::MarsLanguage& language) : + const metkit::mars::MarsLanguage& language) : language_(language), gather_(gather), options_(options) {} void OdbMetadataDecoder::operator()(const std::string& columnName, const std::set& vals) { diff --git a/src/metkit/odb/OdbMetadataDecoder.h b/src/metkit/odb/OdbMetadataDecoder.h index a41cf09c2..282aad0e1 100644 --- a/src/metkit/odb/OdbMetadataDecoder.h +++ b/src/metkit/odb/OdbMetadataDecoder.h @@ -34,7 +34,7 @@ class OdbMetadataDecoder : public odc::api::SpanVisitor { public: OdbMetadataDecoder(eckit::message::MetadataGatherer& gatherer, const eckit::message::GetMetadataOptions& options, - metkit::mars::MarsLanguage& language); + const metkit::mars::MarsLanguage& language); virtual void operator()(const std::string& columnName, const std::set& vals); virtual void operator()(const std::string& columnName, const std::set& vals); @@ -49,7 +49,7 @@ class OdbMetadataDecoder : public odc::api::SpanVisitor { private: // members - metkit::mars::MarsLanguage& language_; + const metkit::mars::MarsLanguage& language_; eckit::message::MetadataGatherer& gather_; eckit::message::GetMetadataOptions options_; }; diff --git a/src/metkit/odb/OdbToRequest.cc b/src/metkit/odb/OdbToRequest.cc index f124d9a14..7d4c2d260 100644 --- a/src/metkit/odb/OdbToRequest.cc +++ b/src/metkit/odb/OdbToRequest.cc @@ -64,7 +64,7 @@ std::vector OdbToRequest::odbToRequest(DataHandle& dh) const { Frame frame; std::vector requests; - MarsLanguage language(verb_); + const MarsLanguage& language = MarsLanguage::get(verb_); while ((frame = reader.next())) { Span span = frame.span(OdbMetadataDecoder::columnNames(), onlyConstantColumns_); diff --git a/tests/test_date.cc b/tests/test_date.cc index 98336fbd9..f6805d978 100644 --- a/tests/test_date.cc +++ b/tests/test_date.cc @@ -31,8 +31,7 @@ using ::eckit::Value; void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); - language.type(name)->expand(values); + MarsLanguage::get("retrieve").type(name)->expand(values); EXPECT_EQUAL(expected, values); } diff --git a/tests/test_expand.cc b/tests/test_expand.cc index d13bdb6e2..d5d708108 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -445,13 +445,11 @@ CASE("test_metkit_expand_multirequest-3") { } void expandKeyThrows(const std::string& key, std::vector values) { - static metkit::mars::MarsLanguage language("retrieve"); - metkit::mars::Type* t = language.type(key); + metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); EXPECT_THROWS_AS(t->expand(values), eckit::BadValue); } void expandKey(const std::string& key, std::vector values, std::vector expected) { - static metkit::mars::MarsLanguage language("retrieve"); - metkit::mars::Type* t = language.type(key); + metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); t->expand(values); EXPECT_EQUAL(expected, values); } diff --git a/tests/test_language.cc b/tests/test_language.cc index 5951032a4..a01405a01 100644 --- a/tests/test_language.cc +++ b/tests/test_language.cc @@ -238,14 +238,14 @@ CASE("check defaults and _clear_defaults") { CASE("check method: isData()") { - EXPECT_EQUAL(MarsLanguage("retrieve").isData("class"), true); - EXPECT_EQUAL(MarsLanguage("retrieve").isData("date"), true); - EXPECT_EQUAL(MarsLanguage("retrieve").isData("time"), true); - EXPECT_EQUAL(MarsLanguage("retrieve").isData("step"), true); - EXPECT_EQUAL(MarsLanguage("retrieve").isData("number"), true); - - EXPECT_EQUAL(MarsLanguage("disseminate").isData("accuracy"), false); - EXPECT_EQUAL(MarsLanguage("disseminate").isData("grid"), false); + EXPECT_EQUAL(MarsLanguage::get("retrieve").isData("class"), true); + EXPECT_EQUAL(MarsLanguage::get("retrieve").isData("date"), true); + EXPECT_EQUAL(MarsLanguage::get("retrieve").isData("time"), true); + EXPECT_EQUAL(MarsLanguage::get("retrieve").isData("step"), true); + EXPECT_EQUAL(MarsLanguage::get("retrieve").isData("number"), true); + + EXPECT_EQUAL(MarsLanguage::get("disseminate").isData("accuracy"), false); + EXPECT_EQUAL(MarsLanguage::get("disseminate").isData("grid"), false); } CASE("check method: flatten()") { @@ -262,7 +262,7 @@ CASE("check method: flatten()") { "500,date=20250717", true); - MarsLanguage("retrieve").flatten(request, output); + MarsLanguage::get("retrieve").flatten(request, output); EXPECT_EQUAL(output.oss.str(), "retrieve,class=od,type=an,stream=oper,levtype=pl,date=20250717,time=1200,step=10,levelist=300,param=" @@ -288,15 +288,15 @@ CASE("check method: flatten()") { CASE("check some types") { { - EXPECT_THROWS(MarsLanguage("read").type("unknown")); + EXPECT_THROWS(MarsLanguage::get("read").type("unknown")); - EXPECT_THROWS(MarsLanguage("retrieve").type("unknown")); + EXPECT_THROWS(MarsLanguage::get("retrieve").type("unknown")); - EXPECT_NO_THROW(MarsLanguage("retrieve").type("_hidden")); + EXPECT_NO_THROW(MarsLanguage::get("retrieve").type("_hidden")); } { - auto language = MarsLanguage("retrieve"); + const auto& language = MarsLanguage::get("retrieve"); auto* type = language.type("class"); EXPECT(dynamic_cast(type) != nullptr); @@ -328,7 +328,7 @@ CASE("check some types") { EXPECT(dynamic_cast(type) != nullptr); } { - auto language = MarsLanguage("archive"); + const auto& language = MarsLanguage::get("archive"); auto* type = language.type("resol"); EXPECT(dynamic_cast(type) != nullptr); diff --git a/tests/test_mars_language.cc b/tests/test_mars_language.cc index f740ecc0d..43358f7d3 100644 --- a/tests/test_mars_language.cc +++ b/tests/test_mars_language.cc @@ -7,7 +7,7 @@ namespace metkit::mars::test { CASE("retrieve_best_match_param_matching") { - const auto language = MarsLanguage("retrieve"); + const auto& language = MarsLanguage::get("retrieve"); // Strict is defaulted to true and this is not matching auto match = language.bestMatch("parameter", {"parameter"}, false, false, false, {}); @@ -17,7 +17,7 @@ CASE("retrieve_best_match_param_matching") { CASE("retrieve_best_match_param_not_matching") { - const auto language = MarsLanguage("retrieve"); + const auto& language = MarsLanguage::get("retrieve"); // Strict is defaulted to true and this is not matching auto match = language.bestMatch("param", {"parameter"}, false, false, false, {}); @@ -30,7 +30,7 @@ CASE("retrieve_best_match_param_not_matching") { }; CASE("retrieve_best_match_param_not_matching_throw") { - const auto language = MarsLanguage("retrieve"); + const auto& language = MarsLanguage::get("retrieve"); // Strict is defaulted to true and this is not matching auto match = language.bestMatch("param", {"parameter"}, true, false, false, {}); @@ -42,7 +42,7 @@ CASE("retrieve_best_match_param_not_matching_throw") { CASE("retrieve_best_match_param_not_matching") { - const auto language = MarsLanguage("retrieve"); + const auto& language = MarsLanguage::get("retrieve"); // Strict is defaulted to true and this is not matching auto match = language.bestMatch("param", {"parameter"}, false, false, true, {}); diff --git a/tests/test_mars_language_strict.cc b/tests/test_mars_language_strict.cc index 53b58dbbc..b56a348ff 100644 --- a/tests/test_mars_language_strict.cc +++ b/tests/test_mars_language_strict.cc @@ -6,17 +6,14 @@ namespace metkit::mars::test { CASE("retrieve_best_match_param_not_matching") { - const auto language = MarsLanguage("retrieve"); - // Strict is defaulted to true and this is not matching - EXPECT_THROWS(language.bestMatch("param", {"parameter"}, false, false, true, {})); + EXPECT_THROWS(MarsLanguage::get("retrieve").bestMatch("param", {"parameter"}, false, false, true, {})); }; CASE("retrieve_best_match_param_matching") { - const auto language = MarsLanguage("retrieve"); - // Strict is defaulted to true and this is not matching - auto match = language.bestMatch("param", {"parameter", "param"}, false, false, true, {}); + const auto& match = + MarsLanguage::get("retrieve").bestMatch("param", {"parameter", "param"}, false, false, true, {}); EXPECT(match == "param"); }; diff --git a/tests/test_obstype.cc b/tests/test_obstype.cc index 3999fd45d..a770afacc 100644 --- a/tests/test_obstype.cc +++ b/tests/test_obstype.cc @@ -28,8 +28,7 @@ using ::eckit::UserError; void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); - language.type(name)->expand(values); + MarsLanguage::get("retrieve").type(name)->expand(values); EXPECT_EQUAL(expected, values); } @@ -44,8 +43,7 @@ CASE("Test Obstype expansions") { assertTypeExpansion("obstype", {"trmm", "qscat"}, {"129", "130", "137", "138"}); { std::vector values{"sd"}; - static MarsLanguage language("retrieve"); - language.type("obstype")->expand(values); + MarsLanguage::get("retrieve").type("obstype")->expand(values); std::vector expected{"121", "122", "123", "124", "210", "212", "213", "214", "216", "217", "218", "51", "53", "54", "55", "56", "57", "59", "60", "61", "62", "63", "65", "71", "72", "73", "75", "138", "139", "153", "155", "211", "240", diff --git a/tests/test_param_axis.cc b/tests/test_param_axis.cc index 9df3e499e..07f675875 100644 --- a/tests/test_param_axis.cc +++ b/tests/test_param_axis.cc @@ -70,10 +70,11 @@ static void test_param_axis(const std::vector& user, const std::vec void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); + const MarsLanguage& language = MarsLanguage::get("retrieve"); MarsRequest req; + MarsRequest ctx; req.setValuesTyped(language.type(name), values); - req = language.expand(req, false, true); + req = language.expand(req, ctx, false, true); EXPECT_EQUAL(expected, req.values(name)); } diff --git a/tests/test_step.cc b/tests/test_step.cc index acd2830b2..73bf83cea 100644 --- a/tests/test_step.cc +++ b/tests/test_step.cc @@ -28,8 +28,7 @@ using ::eckit::BadValue; void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); - language.type(name)->expand(values); + MarsLanguage::get("retrieve").type(name)->expand(values); EXPECT_EQUAL(expected, values); } diff --git a/tests/test_time.cc b/tests/test_time.cc index cadb60bba..68a9d5d80 100644 --- a/tests/test_time.cc +++ b/tests/test_time.cc @@ -32,8 +32,7 @@ using ::eckit::Value; void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); - language.type(name)->expand(values); + MarsLanguage::get("retrieve").type(name)->expand(values); EXPECT_EQUAL(values.size(), expected.size()); EXPECT_EQUAL(expected, values); } diff --git a/tests/test_type_levelist.cc b/tests/test_type_levelist.cc index b525fd5e1..7383b5088 100644 --- a/tests/test_type_levelist.cc +++ b/tests/test_type_levelist.cc @@ -36,8 +36,7 @@ CASE("test_metkit_exists_to-by-list-float") { void assertTypeExpansion(const std::string& name, std::vector values, const std::vector& expected) { - static MarsLanguage language("retrieve"); - language.type(name)->expand(values); + MarsLanguage::get("retrieve").type(name)->expand(values); EXPECT_EQUAL(values, expected); } From 66983a67e7b7efa3686845270f5d693e6556b858 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 09:33:11 +0100 Subject: [PATCH 02/10] METK-183 addressed PR comments --- src/metkit/mars/MarsExpansion.cc | 25 ++++++++++++------------- src/metkit/mars/MarsExpansion.h | 2 -- src/metkit/mars/MarsLanguage.cc | 21 +++++++++++++++------ src/metkit/mars/MarsLanguage.h | 2 -- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index 95f90d63d..d8021cca7 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -19,33 +19,30 @@ namespace metkit::mars { MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {} -const MarsLanguage& MarsExpansion::language(const std::string& verb) { - auto v = MarsLanguage::expandVerb(verb); - return MarsLanguage::get(v); -} - std::vector MarsExpansion::expand(const std::vector& requests) { - MarsRequest ctx; + std::map ctx; std::vector result; result.reserve(requests.size()); // Implement inheritence for (const auto& request : requests) { - auto& lang = language(request.verb()); - result.emplace_back(lang.expand(request, ctx, inherit_, strict_)); + auto verb = MarsLanguage::expandVerb(request.verb()); + auto& ctxForVerb = ctx[verb]; + result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb, inherit_, strict_)); } return result; } std::vector MarsExpansion::expand(const std::vector& requests) { - MarsRequest ctx; + std::map ctx; std::vector result; result.reserve(requests.size()); for (const auto& request : requests) { - auto& lang = language(request.verb()); - result.emplace_back(lang.expand(request, ctx, inherit_, strict_)); + auto verb = MarsLanguage::expandVerb(request.verb()); + auto& ctxForVerb = ctx[verb]; + result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb, inherit_, strict_)); } return result; @@ -53,7 +50,8 @@ std::vector MarsExpansion::expand(const std::vector& r MarsRequest MarsExpansion::expand(const MarsRequest& request) { MarsRequest ctx; - return language(request.verb()).expand(request, ctx, inherit_, strict_); + auto verb = MarsLanguage::expandVerb(request.verb()); + return MarsLanguage::get(verb).expand(request, ctx, inherit_, strict_); } void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) { @@ -61,7 +59,8 @@ void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) } void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) { - language(request.verb()).flatten(request, callback); + auto verb = MarsLanguage::expandVerb(request.verb()); + MarsLanguage::get(verb).flatten(request, callback); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index c8639ee19..c66fa1750 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -68,8 +68,6 @@ class MarsExpansion : public eckit::NonCopyable { private: - const MarsLanguage& language(const std::string& verb); - bool inherit_; bool strict_; }; diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index 157778f7d..e1ea9583b 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include "eckit/config/Resource.h" @@ -453,16 +454,24 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in std::vector params; for (const auto& PP : r.params()) { - auto c = cache_.find(PP); - if (c != cache_.end()) { - paramSet.emplace((*c).second, PP); + std::string p = eckit::StringTools::lower(PP); + bool found = false; + auto it = types_.find(p); + if (it != types_.end()) { + found = true; } else { - std::string p = eckit::StringTools::lower(PP); - paramSet.emplace(cache_[p] = bestMatch(p, keywords_, true, false, true, aliases_), PP); + auto itAlias = aliases_.find(p); + if (itAlias != aliases_.end()) { + p = itAlias->second; + found = true; + } + } + if (!found) { + throw eckit::UserError("Cannot find a definition for '" + PP + "'"); } + paramSet.emplace(p, PP); } - { // sort the parameters, following the AxisOrder for (const auto& k : metkit::hypercube::AxisOrder::instance().axes()) { auto it = paramSet.find(k); diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index f06f6350b..135c3186d 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -94,8 +94,6 @@ class MarsLanguage : private eckit::NonCopyable { std::vector keywords_; std::map aliases_; - - mutable std::map cache_; }; //---------------------------------------------------------------------------------------------------------------------- From d6553fe46df8f88aa98a77b15bfc09f57bcad31f Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 09:37:16 +0100 Subject: [PATCH 03/10] format --- src/metkit/mars/MarsExpansion.cc | 4 ++-- src/metkit/mars/MarsLanguage.cc | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index d8021cca7..920b640c2 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -26,7 +26,7 @@ std::vector MarsExpansion::expand(const std::vector MarsExpansion::expand(const std::vector& r result.reserve(requests.size()); for (const auto& request : requests) { - auto verb = MarsLanguage::expandVerb(request.verb()); + auto verb = MarsLanguage::expandVerb(request.verb()); auto& ctxForVerb = ctx[verb]; result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb, inherit_, strict_)); } diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index e1ea9583b..990e619fe 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -455,15 +455,15 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in for (const auto& PP : r.params()) { std::string p = eckit::StringTools::lower(PP); - bool found = false; - auto it = types_.find(p); + bool found = false; + auto it = types_.find(p); if (it != types_.end()) { found = true; } else { auto itAlias = aliases_.find(p); if (itAlias != aliases_.end()) { - p = itAlias->second; + p = itAlias->second; found = true; } } From fabd5d385aa617e4608b943440035c616b50c795 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 10:25:27 +0100 Subject: [PATCH 04/10] Type constness --- src/metkit/mars/MarsExpansion.cc | 10 +++++----- src/metkit/mars/MarsExpansion.h | 10 +++++----- src/metkit/mars/MarsLanguage.cc | 17 ++++++++++------- src/metkit/mars/MarsRequest.cc | 2 +- src/metkit/mars/MarsRequest.h | 2 +- src/metkit/mars/Parameter.cc | 6 +++--- src/metkit/mars/Parameter.h | 6 +++--- src/metkit/mars/Type.cc | 10 +++++----- src/metkit/mars/Type.h | 10 +++++----- src/metkit/mars/TypeDate.cc | 2 +- src/metkit/mars/TypeDate.h | 2 +- src/metkit/mars/TypeParam.cc | 2 +- src/metkit/mars/TypeParam.h | 2 +- 13 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index 920b640c2..a302b9f4f 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -19,7 +19,7 @@ namespace metkit::mars { MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {} -std::vector MarsExpansion::expand(const std::vector& requests) { +std::vector MarsExpansion::expand(const std::vector& requests) const { std::map ctx; std::vector result; result.reserve(requests.size()); @@ -34,7 +34,7 @@ std::vector MarsExpansion::expand(const std::vector MarsExpansion::expand(const std::vector& requests) { +std::vector MarsExpansion::expand(const std::vector& requests) const { std::map ctx; std::vector result; result.reserve(requests.size()); @@ -48,17 +48,17 @@ std::vector MarsExpansion::expand(const std::vector& r return result; } -MarsRequest MarsExpansion::expand(const MarsRequest& request) { +MarsRequest MarsExpansion::expand(const MarsRequest& request) const { MarsRequest ctx; auto verb = MarsLanguage::expandVerb(request.verb()); return MarsLanguage::get(verb).expand(request, ctx, inherit_, strict_); } -void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) { +void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) const { callback(expand(request)); } -void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) { +void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) const { auto verb = MarsLanguage::expandVerb(request.verb()); MarsLanguage::get(verb).flatten(request, callback); } diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index c66fa1750..e82b0edcb 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -59,12 +59,12 @@ class MarsExpansion : public eckit::NonCopyable { MarsExpansion(bool inherit, bool strict = false); - MarsRequest expand(const MarsRequest&); - std::vector expand(const std::vector&); - std::vector expand(const std::vector&); + MarsRequest expand(const MarsRequest&) const; + std::vector expand(const std::vector&) const; + std::vector expand(const std::vector&) const; - void expand(const MarsRequest&, ExpandCallback&); - void flatten(const MarsRequest&, FlattenCallback&); + void expand(const MarsRequest&, ExpandCallback&) const; + void flatten(const MarsRequest&, FlattenCallback&) const; private: diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index 990e619fe..be5b19aca 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -241,7 +241,14 @@ const MarsLanguage& MarsLanguage::get(const std::string& verb) { if (it != instances.end()) { return *(it->second); } - auto [newIt, inserted] = instances.emplace(verb, new MarsLanguage(verb)); + + auto v = expandVerb(verb); + it = instances.find(v); + if (it != instances.end()) { + return *(it->second); + } + + auto [newIt, inserted] = instances.emplace(v, new MarsLanguage(v)); ASSERT(inserted); return *(newIt->second); } @@ -273,7 +280,6 @@ static bool isnumeric(const std::string& s) { return s.length() > 0; } - std::string MarsLanguage::bestMatch(const std::string& name, const std::vector& values, bool fail, bool quiet, bool fullMatch, const std::map& aliases) { size_t score = (fullMatch ? name.length() : 1); @@ -429,7 +435,6 @@ class TypeHidden : public Type { TypeHidden() : Type("hidden", eckit::Value()) { attach(); } }; - Type* MarsLanguage::type(const std::string& name) const { auto k = types_.find(name); if (k == types_.end()) { @@ -443,8 +448,6 @@ Type* MarsLanguage::type(const std::string& name) const { return k->second; } - -// MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) { MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool inherit, bool strict) const { MarsRequest result(verb_); @@ -468,7 +471,8 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in } } if (!found) { - throw eckit::UserError("Cannot find a definition for '" + PP + "'"); + // fall back to fuzzy matching, governed by METKIT_LANGUAGE_STRICT_MODE + p = bestMatch(p, keywords_, true, false, true, aliases_); } paramSet.emplace(p, PP); } @@ -542,7 +546,6 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in return result; } - const std::string& MarsLanguage::verb() const { return verb_; } diff --git a/src/metkit/mars/MarsRequest.cc b/src/metkit/mars/MarsRequest.cc index 83e283615..bec6dad68 100644 --- a/src/metkit/mars/MarsRequest.cc +++ b/src/metkit/mars/MarsRequest.cc @@ -205,7 +205,7 @@ void MarsRequest::unsetValues(const std::string& name) { } } -void MarsRequest::setValuesTyped(Type* type, const std::vector& values) { +void MarsRequest::setValuesTyped(const Type* type, const std::vector& values) { std::list::iterator i = find(type->name()); if (i != params_.end()) { (*i) = Parameter(values, type); diff --git a/src/metkit/mars/MarsRequest.h b/src/metkit/mars/MarsRequest.h index 6bdd1505a..91bb5a48f 100644 --- a/src/metkit/mars/MarsRequest.h +++ b/src/metkit/mars/MarsRequest.h @@ -111,7 +111,7 @@ class MarsRequest { void dump(std::ostream&, const char* cr = "\n", const char* tab = "\t", bool verb = true) const; - void setValuesTyped(Type*, const std::vector&); + void setValuesTyped(const Type*, const std::vector&); bool filter(const MarsRequest& filter); bool matches(const MarsRequest& filter) const; diff --git a/src/metkit/mars/Parameter.cc b/src/metkit/mars/Parameter.cc index 96f020332..d89704200 100644 --- a/src/metkit/mars/Parameter.cc +++ b/src/metkit/mars/Parameter.cc @@ -45,7 +45,7 @@ Parameter::~Parameter() { type_->detach(); } -Parameter::Parameter(const std::vector& values, Type* type) : type_(type), values_(values) { +Parameter::Parameter(const std::vector& values, const Type* type) : type_(type), values_(values) { if (!type) { type_ = &undefined; } @@ -58,8 +58,8 @@ Parameter::Parameter(const Parameter& other) : type_(other.type_), values_(other } Parameter& Parameter::operator=(const Parameter& other) { - Type* old = type_; - type_ = other.type_; + const Type* old = type_; + type_ = other.type_; type_->attach(); old->detach(); diff --git a/src/metkit/mars/Parameter.h b/src/metkit/mars/Parameter.h index d0b776ac9..446befd6d 100644 --- a/src/metkit/mars/Parameter.h +++ b/src/metkit/mars/Parameter.h @@ -43,7 +43,7 @@ class Parameter { Parameter(); ~Parameter(); - Parameter(const std::vector& values, Type* = 0); + Parameter(const std::vector& values, const Type* = 0); Parameter(const Parameter&); Parameter& operator=(const Parameter&); @@ -58,7 +58,7 @@ class Parameter { void merge(const Parameter& p); - Type& type() const { return *type_; } + const Type& type() const { return *type_; } const std::string& name() const; size_t count() const; @@ -74,7 +74,7 @@ class Parameter { private: // members - Type* type_; + const Type* type_; std::vector values_; }; diff --git a/src/metkit/mars/Type.cc b/src/metkit/mars/Type.cc index 4e71a8742..a3ee5b2c2 100644 --- a/src/metkit/mars/Type.cc +++ b/src/metkit/mars/Type.cc @@ -181,7 +181,7 @@ void Type::set(std::shared_ptr context, const std::vector& void Type::unset(std::shared_ptr context) { unsets_.insert(std::move(context)); } -void Type::patchRequest(MarsRequest& request, const std::vector& values) { +void Type::patchRequest(MarsRequest& request, const std::vector& values) const { // Special case: inheritance from another key. // If the value is of the form _key, then copy values from that key if (values.size() == 1 && values[0][0] == '_') { @@ -314,7 +314,7 @@ void Type::expand(std::vector& values, const MarsRequest& request) } } -void Type::setDefaults(MarsRequest& request) { +void Type::setDefaults(MarsRequest& request) const { bool unset = false; for (const auto& unsetContext : unsets_) { if (unsetContext->matches(request)) { @@ -332,7 +332,7 @@ void Type::setDefaults(MarsRequest& request) { } } -const std::vector& Type::flattenValues(const MarsRequest& request) { +const std::vector& Type::flattenValues(const MarsRequest& request) const { return request.values(name_); } @@ -348,9 +348,9 @@ const std::string& Type::category() const { return category_; } -void Type::pass2(MarsRequest& request) {} +void Type::pass2(MarsRequest& request) const {} -void Type::finalise(MarsRequest& request, bool strict) { +void Type::finalise(MarsRequest& request, bool strict) const { const std::vector& values = request.values(name_, true); if (values.size() == 1 && values[0] == "off") { diff --git a/src/metkit/mars/Type.h b/src/metkit/mars/Type.h index a0c2d3c91..ccee99850 100644 --- a/src/metkit/mars/Type.h +++ b/src/metkit/mars/Type.h @@ -194,14 +194,14 @@ class Type : public eckit::Counted { std::string tidy(const std::string& value, const MarsRequest& request = {}) const; - virtual void setDefaults(MarsRequest& request); + virtual void setDefaults(MarsRequest& request) const; virtual void check(const std::vector& values) const; virtual void clearDefaults(); - virtual void pass2(MarsRequest& request); - virtual void finalise(MarsRequest& request, bool strict); + virtual void pass2(MarsRequest& request) const; + virtual void finalise(MarsRequest& request, bool strict) const; - virtual const std::vector& flattenValues(const MarsRequest& request); + virtual const std::vector& flattenValues(const MarsRequest& request) const; virtual bool flatten() const; virtual bool multiple() const; @@ -250,7 +250,7 @@ class Type : public eckit::Counted { private: // methods virtual void print(std::ostream& out) const = 0; - void patchRequest(MarsRequest& request, const std::vector& values); + void patchRequest(MarsRequest& request, const std::vector& values) const; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/TypeDate.cc b/src/metkit/mars/TypeDate.cc index e98dfc1ac..d026e2ec9 100644 --- a/src/metkit/mars/TypeDate.cc +++ b/src/metkit/mars/TypeDate.cc @@ -122,7 +122,7 @@ TypeDate::TypeDate(const std::string& name, const eckit::Value& settings) : Type filters_["day"] = &filterByDay; } -void TypeDate::pass2(MarsRequest& request) { +void TypeDate::pass2(MarsRequest& request) const { std::vector values = request.values(name_, true); if (values.size() == 1 && values[0] == "-1") { Type::expand(values, request); diff --git a/src/metkit/mars/TypeDate.h b/src/metkit/mars/TypeDate.h index 6d96608b2..f6a310f08 100644 --- a/src/metkit/mars/TypeDate.h +++ b/src/metkit/mars/TypeDate.h @@ -33,7 +33,7 @@ class TypeDate : public Type { private: // methods void print(std::ostream& out) const override; - void pass2(MarsRequest& request) override; + void pass2(MarsRequest& request) const override; bool expand(std::string& value, const MarsRequest& request) const override; }; diff --git a/src/metkit/mars/TypeParam.cc b/src/metkit/mars/TypeParam.cc index 8e8ace359..b731395e1 100644 --- a/src/metkit/mars/TypeParam.cc +++ b/src/metkit/mars/TypeParam.cc @@ -429,7 +429,7 @@ void TypeParam::print(std::ostream& out) const { out << "TypeParam[name=" << name_ << "]"; } -void TypeParam::pass2(MarsRequest& request) { +void TypeParam::pass2(MarsRequest& request) const { pthread_once(&once, init); diff --git a/src/metkit/mars/TypeParam.h b/src/metkit/mars/TypeParam.h index 195457489..351791e29 100644 --- a/src/metkit/mars/TypeParam.h +++ b/src/metkit/mars/TypeParam.h @@ -35,7 +35,7 @@ class TypeParam : public Type { bool firstRule_; void print(std::ostream& out) const override; - void pass2(MarsRequest& request) override; + void pass2(MarsRequest& request) const override; bool expand(std::string& value, const MarsRequest& request) const override; }; From 39ccc0b1ac89ad13ae5f80074aa18ad89daca7af Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 11:12:24 +0100 Subject: [PATCH 05/10] METK-183 fix rust wrapper --- rust/crates/metkit-sys/cpp/MarsRequest.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crates/metkit-sys/cpp/MarsRequest.cc b/rust/crates/metkit-sys/cpp/MarsRequest.cc index 012473018..018beb796 100644 --- a/rust/crates/metkit-sys/cpp/MarsRequest.cc +++ b/rust/crates/metkit-sys/cpp/MarsRequest.cc @@ -88,8 +88,8 @@ std::unique_ptr MarsRequestWrapper::extract(rust::Str catego } std::unique_ptr MarsRequestWrapper::expand(bool inherit, bool strict) const { - metkit::mars::MarsLanguage lang(request_.verb()); - auto expanded = lang.expand(request_, inherit, strict); + metkit::mars::MarsRequest ctx; + auto expanded = metkit::mars::MarsLanguage::get(request_.verb()).expand(request_, ctx, inherit, strict); return std::make_unique(std::move(expanded)); } From 481c160d9c12ab13ea2cd9cfc975bf1c6c69d064 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 14:21:02 +0100 Subject: [PATCH 06/10] MATK-183 MarsLanguage::type returns const --- src/metkit/mars/MarsExpansion.h | 9 ++------ src/metkit/mars/MarsLanguage.cc | 4 ++-- src/metkit/mars/MarsLanguage.h | 7 ++----- src/metkit/mars/TypesFactory.h | 31 ++++++++++++++++------------ src/metkit/odb/OdbMetadataDecoder.cc | 4 ++-- tests/test_expand.cc | 4 ++-- tests/test_language.cc | 24 ++++++++++----------- 7 files changed, 40 insertions(+), 43 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index e82b0edcb..c1a350765 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -14,10 +14,7 @@ /// @date Sep 96 -#ifndef metkit_MarsExpansion_H -#define metkit_MarsExpansion_H - -#include "eckit/memory/NonCopyable.h" +#pragma once #include #include @@ -54,7 +51,7 @@ class ExpandCallback { //---------------------------------------------------------------------------------------------------------------------- -class MarsExpansion : public eckit::NonCopyable { +class MarsExpansion { public: MarsExpansion(bool inherit, bool strict = false); @@ -75,5 +72,3 @@ class MarsExpansion : public eckit::NonCopyable { //---------------------------------------------------------------------------------------------------------------------- } // namespace metkit::mars - -#endif diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index be5b19aca..6c7e5a3ee 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -435,7 +435,7 @@ class TypeHidden : public Type { TypeHidden() : Type("hidden", eckit::Value()) { attach(); } }; -Type* MarsLanguage::type(const std::string& name) const { +const Type* MarsLanguage::type(const std::string& name) const { auto k = types_.find(name); if (k == types_.end()) { if (name[0] == '_') { @@ -559,7 +559,7 @@ void MarsLanguage::flatten(const MarsRequest& request, const std::vectorflatten()) { flatten(request, params, i + 1, result, callback); return; diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index 135c3186d..9d723e919 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -22,11 +22,8 @@ #include #include -#include "eckit/memory/NonCopyable.h" - #include "metkit/mars/MarsRequest.h" - namespace metkit::mars { class Context; @@ -43,7 +40,7 @@ enum class ModifierType { //---------------------------------------------------------------------------------------------------------------------- -class MarsLanguage : private eckit::NonCopyable { +class MarsLanguage { public: // methods @@ -57,7 +54,7 @@ class MarsLanguage : private eckit::NonCopyable { void flatten(const MarsRequest& request, FlattenCallback& callback) const; - Type* type(const std::string& name) const; + const Type* type(const std::string& name) const; bool isData(const std::string& keyword) const; diff --git a/src/metkit/mars/TypesFactory.h b/src/metkit/mars/TypesFactory.h index 6672c9dc9..9ec5fac47 100644 --- a/src/metkit/mars/TypesFactory.h +++ b/src/metkit/mars/TypesFactory.h @@ -13,41 +13,49 @@ /// @author Tiago Quintino /// @date April 2016 -#ifndef metkit_TypesFactory_H -#define metkit_TypesFactory_H +#pragma once #include -#include "eckit/memory/NonCopyable.h" #include "eckit/types/Types.h" namespace eckit { class Value; } -namespace metkit { -namespace mars { +namespace metkit::mars { class Type; class TypesFactory; //---------------------------------------------------------------------------------------------------------------------- - -class TypesRegistry : private eckit::NonCopyable { - eckit::Mutex mutex_; - std::map m_; +class TypesRegistry { public: static TypesRegistry& instance(); + TypesRegistry(const TypesRegistry&) = delete; + TypesRegistry(TypesRegistry&&) = delete; + TypesRegistry& operator=(const TypesRegistry&) = delete; + TypesRegistry& operator=(TypesRegistry&&) = delete; + void add(const std::string& name, TypesFactory* f); void remove(const std::string& name); Type* build(const std::string& keyword, const eckit::Value&); void list(std::ostream& s); + +private: // methods + + TypesRegistry() = default; + +private: // members + + eckit::Mutex mutex_; + std::map m_; }; /// A self-registering factory for producing TypesFactory instances @@ -86,7 +94,4 @@ class TypeBuilder : public TypesFactory { //---------------------------------------------------------------------------------------------------------------------- -} // namespace mars -} // namespace metkit - -#endif +} // namespace metkit::mars diff --git a/src/metkit/odb/OdbMetadataDecoder.cc b/src/metkit/odb/OdbMetadataDecoder.cc index aef237fa5..d818c8b9a 100644 --- a/src/metkit/odb/OdbMetadataDecoder.cc +++ b/src/metkit/odb/OdbMetadataDecoder.cc @@ -76,8 +76,8 @@ void OdbMetadataDecoder::visit(const std::string& columnName, const std::set& auto mapitr = OdbColumnNameMapping::instance().table().find(columnName); ASSERT(mapitr != OdbColumnNameMapping::instance().table().end()); - std::string keyword = eckit::StringTools::lower(mapitr->second); - metkit::mars::Type* t = language.type(keyword); + std::string keyword = eckit::StringTools::lower(mapitr->second); + const metkit::mars::Type* t = language.type(keyword); ASSERT(options_.valueRepresentation == eckit::message::ValueRepresentation::String); diff --git a/tests/test_expand.cc b/tests/test_expand.cc index d5d708108..d1f5bfed2 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -445,11 +445,11 @@ CASE("test_metkit_expand_multirequest-3") { } void expandKeyThrows(const std::string& key, std::vector values) { - metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); + const metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); EXPECT_THROWS_AS(t->expand(values), eckit::BadValue); } void expandKey(const std::string& key, std::vector values, std::vector expected) { - metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); + const metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); t->expand(values); EXPECT_EQUAL(expected, values); } diff --git a/tests/test_language.cc b/tests/test_language.cc index a01405a01..1523a62ba 100644 --- a/tests/test_language.cc +++ b/tests/test_language.cc @@ -298,40 +298,40 @@ CASE("check some types") { { const auto& language = MarsLanguage::get("retrieve"); - auto* type = language.type("class"); - EXPECT(dynamic_cast(type) != nullptr); + const auto* type = language.type("class"); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("param"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("expver"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("domain"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("date"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("grid"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); EXPECT_EQUAL(type->multiple(), true); type = language.type("area"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); EXPECT_EQUAL(type->multiple(), true); type = language.type("accuracy"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); type = language.type("resol"); - EXPECT(dynamic_cast(type) != nullptr); + EXPECT(dynamic_cast(type) != nullptr); } { const auto& language = MarsLanguage::get("archive"); - auto* type = language.type("resol"); - EXPECT(dynamic_cast(type) != nullptr); + const auto* type = language.type("resol"); + EXPECT(dynamic_cast(type) != nullptr); EXPECT_THROWS_AS(language.type("grid"), eckit::SeriousBug); EXPECT_THROWS_AS(language.type("area"), eckit::SeriousBug); From ab14f35072bffd91d29cef8598952bb263e6580f Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 17 Sep 2026 14:34:35 +0100 Subject: [PATCH 07/10] METK-183 additional inheritance test --- tests/test_expand.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_expand.cc b/tests/test_expand.cc index d1f5bfed2..d6d01c09d 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -444,6 +444,13 @@ CASE("test_metkit_expand_multirequest-3") { expand(text, std::vector{expected1, expected2}); } +CASE("test_metkit_expand_multirequest-4") { + const std::string text = "ret,class=dm,type=fc,levtype=ml,date=-2\nret,date=-1"; + std::string expectedStr = + "retrieve,class=dm,type=fc,stream=oper,levtype=ml,time=0,step=0,expver=0001,domain=g,levelist=1,param=129"; + expand(text, {expectedStr, expectedStr}, false, {{-2}, {-1}}); +} + void expandKeyThrows(const std::string& key, std::vector values) { const metkit::mars::Type* t = metkit::mars::MarsLanguage::get("retrieve").type(key); EXPECT_THROWS_AS(t->expand(values), eckit::BadValue); From a92d375896ccc811e63928ac82987827ec9f962b Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 18 Sep 2026 22:42:14 +0100 Subject: [PATCH 08/10] METK-183 introduced ExpansionContext class --- src/metkit/mars/MarsExpansion.cc | 6 +++--- src/metkit/mars/MarsLanguage.cc | 30 +++++++++++++++++++++++++++--- src/metkit/mars/MarsLanguage.h | 17 ++++++++++++++++- tests/test_param_axis.cc | 2 +- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index a302b9f4f..f63ec14ea 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -20,7 +20,7 @@ namespace metkit::mars { MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {} std::vector MarsExpansion::expand(const std::vector& requests) const { - std::map ctx; + std::map ctx; std::vector result; result.reserve(requests.size()); @@ -35,7 +35,7 @@ std::vector MarsExpansion::expand(const std::vector MarsExpansion::expand(const std::vector& requests) const { - std::map ctx; + std::map ctx; std::vector result; result.reserve(requests.size()); @@ -49,7 +49,7 @@ std::vector MarsExpansion::expand(const std::vector& r } MarsRequest MarsExpansion::expand(const MarsRequest& request) const { - MarsRequest ctx; + ExpansionContext ctx; auto verb = MarsLanguage::expandVerb(request.verb()); return MarsLanguage::get(verb).expand(request, ctx, inherit_, strict_); } diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index 6c7e5a3ee..f837ada8b 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -59,6 +59,31 @@ static void init() { namespace metkit::mars { //---------------------------------------------------------------------------------------------------------------------- + +ExpansionContext::ExpansionContext(const MarsRequest& request) { + for (const auto& param : request.parameters()) { + values_[param.name()] = param.values(); + } +} + +ExpansionContext& ExpansionContext::operator=(ExpansionContext&& other) { + values_ = std::move(other.values_); + return *this; +} + +bool ExpansionContext::has(const std::string& key) const { + return values_.find(key) != values_.end(); +} + +const std::vector& ExpansionContext::values(const std::string& key) const { + static const std::vector empty; + auto it = values_.find(key); + if (it != values_.end()) { + return it->second; + } + return empty; +} + void MarsLanguage::parseModifier(ModifierType typ, std::shared_ptr ctx, size_t maxIndex, const eckit::Value& mod) { eckit::Value keys; @@ -448,7 +473,7 @@ const Type* MarsLanguage::type(const std::string& name) const { return k->second; } -MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool inherit, bool strict) const { +MarsRequest MarsLanguage::expand(const MarsRequest& r, ExpansionContext& ctx, bool inherit, bool strict) const { MarsRequest result(verb_); try { @@ -496,7 +521,6 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in const std::string& s = eckit::StringTools::lower(values[0]); if (s == "off") { result.unsetValues(p); - ctx.unsetValues(p); continue; } if (s == "all" && type(p)->multiple()) { @@ -541,7 +565,7 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, MarsRequest& ctx, bool in throw eckit::UserError(oss.str()); } if (inherit) { - ctx = result; + ctx = ExpansionContext(result); } return result; } diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index 9d723e919..dc8e58e14 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -40,6 +40,21 @@ enum class ModifierType { //---------------------------------------------------------------------------------------------------------------------- +class ExpansionContext { +public: + + ExpansionContext() = default; + ExpansionContext(const MarsRequest& request); + ExpansionContext& operator=(ExpansionContext&& other); + + bool has(const std::string& key) const; + const std::vector& values(const std::string& key) const; + +private: + + std::map> values_; +}; + class MarsLanguage { public: // methods @@ -48,7 +63,7 @@ class MarsLanguage { ~MarsLanguage(); - MarsRequest expand(const MarsRequest& r, MarsRequest& ctx, bool inherit, bool strict) const; + MarsRequest expand(const MarsRequest& r, ExpansionContext& ctx, bool inherit, bool strict) const; const std::string& verb() const; diff --git a/tests/test_param_axis.cc b/tests/test_param_axis.cc index 07f675875..29b65a380 100644 --- a/tests/test_param_axis.cc +++ b/tests/test_param_axis.cc @@ -72,7 +72,7 @@ void assertTypeExpansion(const std::string& name, std::vector value const std::vector& expected) { const MarsLanguage& language = MarsLanguage::get("retrieve"); MarsRequest req; - MarsRequest ctx; + ExpansionContext ctx; req.setValuesTyped(language.type(name), values); req = language.expand(req, ctx, false, true); EXPECT_EQUAL(expected, req.values(name)); From c86defff011aa0ddf324e440d81b9dfc6db69737 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Sat, 19 Sep 2026 00:01:57 +0100 Subject: [PATCH 09/10] METK-183 fixed MarsExpansion inheritance over multiple calls + reset --- rust/crates/metkit-sys/cpp/MarsRequest.cc | 2 +- src/metkit/mars/MarsExpansion.cc | 42 ++++++++------ src/metkit/mars/MarsExpansion.h | 21 +++++-- src/metkit/mars/MarsLanguage.h | 6 +- tests/test_expand.cc | 67 +++++++++++++++++++++++ 5 files changed, 114 insertions(+), 24 deletions(-) diff --git a/rust/crates/metkit-sys/cpp/MarsRequest.cc b/rust/crates/metkit-sys/cpp/MarsRequest.cc index 018beb796..f28c7997d 100644 --- a/rust/crates/metkit-sys/cpp/MarsRequest.cc +++ b/rust/crates/metkit-sys/cpp/MarsRequest.cc @@ -88,7 +88,7 @@ std::unique_ptr MarsRequestWrapper::extract(rust::Str catego } std::unique_ptr MarsRequestWrapper::expand(bool inherit, bool strict) const { - metkit::mars::MarsRequest ctx; + metkit::mars::ExpansionContext ctx; auto expanded = metkit::mars::MarsLanguage::get(request_.verb()).expand(request_, ctx, inherit, strict); return std::make_unique(std::move(expanded)); } diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index f63ec14ea..910fee741 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -12,57 +12,67 @@ #include "metkit/mars/MarsLanguage.h" - namespace metkit::mars { //---------------------------------------------------------------------------------------------------------------------- MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {} -std::vector MarsExpansion::expand(const std::vector& requests) const { - std::map ctx; +void MarsExpansion::reset() { + ctx_.clear(); +} + +std::vector MarsExpansion::expand(const std::vector& requests) { std::vector result; result.reserve(requests.size()); // Implement inheritence for (const auto& request : requests) { - auto verb = MarsLanguage::expandVerb(request.verb()); - auto& ctxForVerb = ctx[verb]; - result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb, inherit_, strict_)); + auto verb = MarsLanguage::expandVerb(request.verb()); + result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_)); } return result; } -std::vector MarsExpansion::expand(const std::vector& requests) const { - std::map ctx; +std::vector MarsExpansion::expand(const std::vector& requests) { std::vector result; result.reserve(requests.size()); for (const auto& request : requests) { - auto verb = MarsLanguage::expandVerb(request.verb()); - auto& ctxForVerb = ctx[verb]; - result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb, inherit_, strict_)); + auto verb = MarsLanguage::expandVerb(request.verb()); + result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_)); } return result; } -MarsRequest MarsExpansion::expand(const MarsRequest& request) const { - ExpansionContext ctx; +MarsRequest MarsExpansion::expand(const MarsRequest& request) { auto verb = MarsLanguage::expandVerb(request.verb()); - return MarsLanguage::get(verb).expand(request, ctx, inherit_, strict_); + return MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_); } -void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) const { +void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) { callback(expand(request)); } -void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) const { +void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) { auto verb = MarsLanguage::expandVerb(request.verb()); MarsLanguage::get(verb).flatten(request, callback); } +ExpansionContext& MarsExpansion::ctxForVerb(const std::string& verb) { + static ExpansionContext dummy; + if (!inherit_) { + return dummy; + } + auto it = ctx_.find(verb); + if (it == ctx_.end()) { + it = ctx_.emplace(verb, new ExpansionContext{}).first; + } + return *(it->second); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace metkit::mars diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index c1a350765..41beef6ae 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -25,7 +26,7 @@ namespace metkit::mars { -class MarsLanguage; +class ExpansionContext; //---------------------------------------------------------------------------------------------------------------------- @@ -56,17 +57,25 @@ class MarsExpansion { MarsExpansion(bool inherit, bool strict = false); - MarsRequest expand(const MarsRequest&) const; - std::vector expand(const std::vector&) const; - std::vector expand(const std::vector&) const; + void reset(); - void expand(const MarsRequest&, ExpandCallback&) const; - void flatten(const MarsRequest&, FlattenCallback&) const; + MarsRequest expand(const MarsRequest&); + std::vector expand(const std::vector&); + std::vector expand(const std::vector&); + + void expand(const MarsRequest&, ExpandCallback&); + void flatten(const MarsRequest&, FlattenCallback&); + +private: + + ExpansionContext& ctxForVerb(const std::string& verb); private: bool inherit_; bool strict_; + + std::map ctx_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index dc8e58e14..8841fbc5b 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -60,9 +60,13 @@ class MarsLanguage { public: // methods MarsLanguage(const std::string& verb); - ~MarsLanguage(); + MarsLanguage(const MarsLanguage&) = delete; + MarsLanguage(MarsLanguage&&) = delete; + MarsLanguage& operator=(const MarsLanguage&) = delete; + MarsLanguage& operator=(MarsLanguage&&) = delete; + MarsRequest expand(const MarsRequest& r, ExpansionContext& ctx, bool inherit, bool strict) const; const std::string& verb() const; diff --git a/tests/test_expand.cc b/tests/test_expand.cc index d6d01c09d..7871a6104 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -445,6 +445,73 @@ CASE("test_metkit_expand_multirequest-3") { } CASE("test_metkit_expand_multirequest-4") { + + MarsExpansion exp(true); + + const char* text1 = + "retrieve,accuracy=16,area=14.8/-19.6/-14.5/19.8,class=od,date=20230810,expver=1,grid=0.09/0.09,levelist=1/" + "to/137,levtype=ml,number=-1,param=z,process=local,rotation=-78.8/-61.0,step=000,stream=scda,time=18," + "type=an,target=\"reference.ect1qF.data\""; + const char* text2 = "ret,date=20230811,rotation=off,target=\"out\""; + + std::istringstream in1(text1); + std::istringstream in2(text2); + MarsParser parser1(in1); + MarsParser parser2(in2); + auto r1 = parser1.parse(); + auto r2 = parser2.parse(); + std::vector v1 = exp.expand(r1); + std::vector v2 = exp.expand(r2); + exp.reset(); + std::vector v2_after_reset = exp.expand(r2); + + EXPECT_EQUAL(v1.size(), 1); + EXPECT_EQUAL(v2.size(), 1); + EXPECT_EQUAL(v2_after_reset.size(), 1); + + const char* expected1 = + "RETRIEVE,CLASS=OD,TYPE=AN,STREAM=SCDA,EXPVER=0001,REPRES=SH,LEVTYPE=ML,LEVELIST=1/2/3/4/5/6/7/8/9/10/11/" + "12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/28/29/30/31/32/33/34/35/36/37/38/39/40/41/42/43/44/45/46/" + "47/48/49/50/51/52/53/54/55/56/57/58/59/60/61/62/63/64/65/66/67/68/69/70/71/72/73/74/75/76/77/78/79/80/81/" + "82/83/84/85/86/87/88/89/90/91/92/93/94/95/96/97/98/99/100/101/102/103/104/105/106/107/108/109/110/111/112/" + "113/114/115/116/117/118/119/120/121/122/123/124/125/126/127/128/129/130/131/132/133/134/135/136/137," + "PARAM=129,DATE=20230810,TIME=1800,STEP=0,DOMAIN=G,TARGET=reference.ect1qF.data,RESOL=AUTO,ACCURACY=16," + "AREA=14.8/-19.6/-14.5/19.8,ROTATION=-78.8/-61,GRID=.09/.09,PROCESS=LOCAL"; + const char* expected2 = + "RETRIEVE,CLASS=OD,TYPE=AN,STREAM=SCDA,EXPVER=0001,REPRES=SH,LEVTYPE=ML,LEVELIST=1/2/3/4/5/6/7/8/9/10/11/" + "12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/28/29/30/31/32/33/34/35/36/37/38/39/40/41/42/43/44/45/46/" + "47/48/49/50/51/52/53/54/55/56/57/58/59/60/61/62/63/64/65/66/67/68/69/70/71/72/73/74/75/76/77/78/79/80/81/" + "82/83/84/85/86/87/88/89/90/91/92/93/94/95/96/97/98/99/100/101/102/103/104/105/106/107/108/109/110/111/112/" + "113/114/115/116/117/118/119/120/121/122/123/124/125/126/127/128/129/130/131/132/133/134/135/136/137," + "PARAM=129,DATE=20230811,TIME=1800,STEP=0,DOMAIN=G,TARGET=out,RESOL=AUTO,ACCURACY=16," + "AREA=14.8/-19.6/-14.5/19.8,GRID=.09/.09,PROCESS=LOCAL"; + const char* expected2_after_reset = + "retrieve,target=out,class=od,type=an,stream=oper,levtype=pl,date=20230811,time=1200,step=0,expver=0001,domain=" + "g,levelist=1000/850/700/500/400/300,param=129"; + + std::set ignore; + std::string tt = eckit::StringTools::lower(text1); + for (const auto& i : ignore_) { + auto idx = tt.find(i); + if (idx == std::string::npos) { + ignore.insert(i); + } + } + + ExpectedRequest out1; + ExpectedRequest out2; + ExpectedRequest out2_after_reset; + + parse(expected1, out1); + parse(expected2, out2); + parse(expected2_after_reset, out2_after_reset); + + expand(v1[0], out1, ignore); + expand(v2[0], out2, ignore); + expand(v2_after_reset[0], out2_after_reset, ignore); +} + +CASE("test_metkit_expand_multirequest-5") { const std::string text = "ret,class=dm,type=fc,levtype=ml,date=-2\nret,date=-1"; std::string expectedStr = "retrieve,class=dm,type=fc,stream=oper,levtype=ml,time=0,step=0,expver=0001,domain=g,levelist=1,param=129"; From bbda19c01723316767e30b3e469f374f9e24adcd Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Sat, 19 Sep 2026 00:15:39 +0100 Subject: [PATCH 10/10] METK-183 fixed MarsExpansion inheritance over multiple calls + reset --- src/metkit/mars/MarsExpansion.cc | 6 +----- src/metkit/mars/MarsExpansion.h | 6 ++---- src/metkit/mars/MarsLanguage.cc | 5 +++++ src/metkit/mars/MarsLanguage.h | 1 + tests/test_expand.cc | 1 + 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index 910fee741..2b6696583 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -66,11 +66,7 @@ ExpansionContext& MarsExpansion::ctxForVerb(const std::string& verb) { if (!inherit_) { return dummy; } - auto it = ctx_.find(verb); - if (it == ctx_.end()) { - it = ctx_.emplace(verb, new ExpansionContext{}).first; - } - return *(it->second); + return ctx_[verb]; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index 41beef6ae..3dfa07c1f 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -17,17 +17,15 @@ #pragma once #include -#include #include #include +#include "metkit/mars/MarsLanguage.h" #include "metkit/mars/MarsParsedRequest.h" #include "metkit/mars/MarsRequest.h" namespace metkit::mars { -class ExpansionContext; - //---------------------------------------------------------------------------------------------------------------------- class FlattenCallback { @@ -75,7 +73,7 @@ class MarsExpansion { bool inherit_; bool strict_; - std::map ctx_; + std::map ctx_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index f837ada8b..9894f82ac 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -84,6 +84,10 @@ const std::vector& ExpansionContext::values(const std::string& key) return empty; } +void ExpansionContext::unset(const std::string& key) { + values_.erase(key); +} + void MarsLanguage::parseModifier(ModifierType typ, std::shared_ptr ctx, size_t maxIndex, const eckit::Value& mod) { eckit::Value keys; @@ -521,6 +525,7 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, ExpansionContext& ctx, bo const std::string& s = eckit::StringTools::lower(values[0]); if (s == "off") { result.unsetValues(p); + ctx.unset(p); continue; } if (s == "all" && type(p)->multiple()) { diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index 8841fbc5b..339c12e23 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -49,6 +49,7 @@ class ExpansionContext { bool has(const std::string& key) const; const std::vector& values(const std::string& key) const; + void unset(const std::string& key); private: diff --git a/tests/test_expand.cc b/tests/test_expand.cc index 7871a6104..4fc217270 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -508,6 +508,7 @@ CASE("test_metkit_expand_multirequest-4") { expand(v1[0], out1, ignore); expand(v2[0], out2, ignore); + EXPECT(!v2[0].has("rotation")); expand(v2_after_reset[0], out2_after_reset, ignore); }