Skip to content

METK-183 MarsLanguage immutable + lazy loading + caching #270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danovaro merged 10 commits into from
Sep 18, 2026
Merged

METK-183 MarsLanguage immutable + lazy loading + caching #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0b17662
METK-183 MarsLanguage immutable + lazy loading + caching
danovaro Sep 17, 2026
66983a6
METK-183 addressed PR comments
danovaro Sep 17, 2026
d6553fe
format
danovaro Sep 17, 2026
fabd5d3
Type constness
danovaro Sep 17, 2026
39ccc0b
METK-183 fix rust wrapper
danovaro Sep 17, 2026
481c160
MATK-183 MarsLanguage::type returns const
danovaro Sep 17, 2026
ab14f35
METK-183 additional inheritance test
danovaro Sep 17, 2026
a92d375
METK-183 introduced ExpansionContext class
danovaro Sep 18, 2026
c86deff
METK-183 fixed MarsExpansion inheritance over multiple calls + reset
danovaro Sep 18, 2026
bbda19c
METK-183 fixed MarsExpansion inheritance over multiple calls + reset
danovaro Sep 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions rust/crates/metkit-sys/cpp/MarsRequest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ std::unique_ptr<MarsRequestWrapper> MarsRequestWrapper::extract(rust::Str catego
}

std::unique_ptr<MarsRequestWrapper> MarsRequestWrapper::expand(bool inherit, bool strict) const {
metkit::mars::MarsLanguage lang(request_.verb());
auto expanded = lang.expand(request_, inherit, strict);
metkit::mars::ExpansionContext ctx;
auto expanded = metkit::mars::MarsLanguage::get(request_.verb()).expand(request_, ctx, inherit, strict);
return std::make_unique<MarsRequestWrapper>(std::move(expanded));
}

Expand Down
45 changes: 17 additions & 28 deletions src/metkit/mars/MarsExpansion.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,45 +12,24 @@

#include "metkit/mars/MarsLanguage.h"


namespace metkit::mars {

//----------------------------------------------------------------------------------------------------------------------

MarsExpansion::MarsExpansion(bool inherit, bool strict) : inherit_(inherit), strict_(strict) {}

MarsExpansion::~MarsExpansion() {
for (auto& language : languages_) {
delete language.second;
}
}

void MarsExpansion::reset() {
for (auto& language : languages_) {
language.second->reset();
}
}

MarsLanguage& MarsExpansion::language(const std::string& verb) {
auto v = MarsLanguage::expandVerb(verb);

if (auto j = languages_.find(v); j != languages_.end()) {
return *(*j).second;
}

auto j = languages_.emplace(v, new MarsLanguage(v)).first;
return *(*j).second;
ctx_.clear();
}


std::vector<MarsRequest> MarsExpansion::expand(const std::vector<MarsParsedRequest>& requests) {
std::vector<MarsRequest> result;
result.reserve(requests.size());

// Implement inheritence
for (const auto& request : requests) {
auto& lang = language(request.verb());
result.emplace_back(lang.expand(request, inherit_, strict_));
auto verb = MarsLanguage::expandVerb(request.verb());
result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_));
}

return result;
Expand All @@ -61,23 +40,33 @@ std::vector<MarsRequest> MarsExpansion::expand(const std::vector<MarsRequest>& r
result.reserve(requests.size());

for (const auto& request : requests) {
auto& lang = language(request.verb());
result.emplace_back(lang.expand(request, inherit_, strict_));
auto verb = MarsLanguage::expandVerb(request.verb());
result.emplace_back(MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_));
}

return result;
}

MarsRequest MarsExpansion::expand(const MarsRequest& request) {
return language(request.verb()).expand(request, inherit_, strict_);
auto verb = MarsLanguage::expandVerb(request.verb());
return MarsLanguage::get(verb).expand(request, ctxForVerb(verb), inherit_, strict_);
}

void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) {
callback(expand(request));
}

void MarsExpansion::flatten(const MarsRequest& request, FlattenCallback& callback) {
language(request.verb()).flatten(request, callback);
auto verb = MarsLanguage::expandVerb(request.verb());
MarsLanguage::get(verb).flatten(request, callback);
}

ExpansionContext& MarsExpansion::ctxForVerb(const std::string& verb) {
static ExpansionContext dummy;
if (!inherit_) {
return dummy;
}
return ctx_[verb];
}

//----------------------------------------------------------------------------------------------------------------------
Expand Down
20 changes: 8 additions & 12 deletions src/metkit/mars/MarsExpansion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,18 @@

/// @date Sep 96

#ifndef metkit_MarsExpansion_H
#define metkit_MarsExpansion_H

#include "eckit/memory/NonCopyable.h"
#pragma once

#include <map>
#include <string>
#include <vector>

#include "metkit/mars/MarsLanguage.h"
#include "metkit/mars/MarsParsedRequest.h"
#include "metkit/mars/MarsRequest.h"

namespace metkit::mars {

class MarsLanguage;

//----------------------------------------------------------------------------------------------------------------------

class FlattenCallback {
Expand All @@ -54,11 +50,10 @@ class ExpandCallback {

//----------------------------------------------------------------------------------------------------------------------

class MarsExpansion : public eckit::NonCopyable {
class MarsExpansion {
public:

MarsExpansion(bool inherit, bool strict = false);
~MarsExpansion();

void reset();

Expand All @@ -71,15 +66,16 @@ class MarsExpansion : public eckit::NonCopyable {

private:

MarsLanguage& language(const std::string& verb);
ExpansionContext& ctxForVerb(const std::string& verb);

private:

std::map<std::string, MarsLanguage*> languages_;
bool inherit_;
bool strict_;

std::map<std::string, ExpansionContext> ctx_;
};

//----------------------------------------------------------------------------------------------------------------------

} // namespace metkit::mars

#endif
105 changes: 77 additions & 28 deletions src/metkit/mars/MarsLanguage.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include <algorithm>
#include <fstream>
#include <mutex>
#include <optional>

#include "eckit/config/Resource.h"
Expand Down Expand Up @@ -58,6 +59,35 @@ static void init() {
namespace metkit::mars {

//----------------------------------------------------------------------------------------------------------------------

ExpansionContext::ExpansionContext(const MarsRequest& request) {
for (const auto& param : request.parameters()) {
values_[param.name()] = param.values();
}
}

ExpansionContext& ExpansionContext::operator=(ExpansionContext&& other) {
values_ = std::move(other.values_);
return *this;
}

bool ExpansionContext::has(const std::string& key) const {
return values_.find(key) != values_.end();
}

const std::vector<std::string>& ExpansionContext::values(const std::string& key) const {
static const std::vector<std::string> empty;
auto it = values_.find(key);
if (it != values_.end()) {
return it->second;
}
return empty;
}

void ExpansionContext::unset(const std::string& key) {
values_.erase(key);
}

void MarsLanguage::parseModifier(ModifierType typ, std::shared_ptr<Context> ctx, size_t maxIndex,
const eckit::Value& mod) {
eckit::Value keys;
Expand Down Expand Up @@ -231,14 +261,25 @@ MarsLanguage::~MarsLanguage() {
}
}

eckit::PathName MarsLanguage::languageYamlFile() {
return metkit::LibMetkit::languageYamlFile();
}
const MarsLanguage& MarsLanguage::get(const std::string& verb) {
static std::mutex mutex;
Comment thread
danovaro marked this conversation as resolved.
static std::map<std::string, MarsLanguage*> instances;

void MarsLanguage::reset() {
for (auto& [k, t] : types_) {
t->reset();
std::lock_guard lock(mutex);
auto it = instances.find(verb);
if (it != instances.end()) {
return *(it->second);
}

auto v = expandVerb(verb);
it = instances.find(v);
if (it != instances.end()) {
return *(it->second);
}

auto [newIt, inserted] = instances.emplace(v, new MarsLanguage(v));
ASSERT(inserted);
return *(newIt->second);
}

eckit::Value MarsLanguage::jsonFile(const std::string& name) {
Expand Down Expand Up @@ -268,7 +309,6 @@ static bool isnumeric(const std::string& s) {
return s.length() > 0;
}


std::string MarsLanguage::bestMatch(const std::string& name, const std::vector<std::string>& values, bool fail,
bool quiet, bool fullMatch, const std::map<std::string, std::string>& aliases) {
size_t score = (fullMatch ? name.length() : 1);
Expand Down Expand Up @@ -424,8 +464,7 @@ class TypeHidden : public Type {
TypeHidden() : Type("hidden", eckit::Value()) { attach(); }
};


Type* MarsLanguage::type(const std::string& name) const {
const Type* MarsLanguage::type(const std::string& name) const {
auto k = types_.find(name);
if (k == types_.end()) {
if (name[0] == '_') {
Expand All @@ -438,8 +477,7 @@ Type* MarsLanguage::type(const std::string& name) const {
return k->second;
}


MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) {
MarsRequest MarsLanguage::expand(const MarsRequest& r, ExpansionContext& ctx, bool inherit, bool strict) const {
MarsRequest result(verb_);

try {
Expand All @@ -448,16 +486,25 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict
std::vector<std::string> params;

for (const auto& PP : r.params()) {
auto c = cache_.find(PP);
if (c != cache_.end()) {
paramSet.emplace((*c).second, PP);
std::string p = eckit::StringTools::lower(PP);
bool found = false;
auto it = types_.find(p);
if (it != types_.end()) {
found = true;
}
else {
std::string p = eckit::StringTools::lower(PP);
paramSet.emplace(cache_[p] = bestMatch(p, keywords_, true, false, true, aliases_), PP);
auto itAlias = aliases_.find(p);
if (itAlias != aliases_.end()) {
p = itAlias->second;
found = true;
}
}
if (!found) {
// fall back to fuzzy matching, governed by METKIT_LANGUAGE_STRICT_MODE
p = bestMatch(p, keywords_, true, false, true, aliases_);
}
paramSet.emplace(p, PP);
}

{ // sort the parameters, following the AxisOrder
for (const auto& k : metkit::hypercube::AxisOrder::instance().axes()) {
auto it = paramSet.find(k);
Expand All @@ -478,7 +525,7 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict
const std::string& s = eckit::StringTools::lower(values[0]);
if (s == "off") {
result.unsetValues(p);
type(p)->reset();
ctx.unset(p);
continue;
}
if (s == "all" && type(p)->multiple()) {
Expand All @@ -496,14 +543,14 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict
if (inherit) {
for (const auto& [k, t] : typesByAxisOrder_) {
if (t != nullptr && result.countValues(k) == 0) {
t->setDefaults(result);
if (ctx.has(k)) {
result.setValuesTyped(t, ctx.values(k));
}
else {
t->setDefaults(result);
Comment thread
Copilot marked this conversation as resolved.
}
}
}

result.getParams(params);
for (std::vector<std::string>::const_iterator k = params.begin(); k != params.end(); ++k) {
type(*k)->setInheritance(result.values(*k));
}
}

result.getParams(params);
Expand All @@ -522,24 +569,26 @@ MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict
oss << e.what() << " request=" << r << ", expanded=" << result;
throw eckit::UserError(oss.str());
}
if (inherit) {
ctx = ExpansionContext(result);
}
return result;
}


const std::string& MarsLanguage::verb() const {
return verb_;
}

void MarsLanguage::flatten(const MarsRequest& request, const std::vector<std::string>& params, size_t i,
MarsRequest& result, FlattenCallback& callback) {
MarsRequest& result, FlattenCallback& callback) const {
if (i == params.size()) {
callback(result);
return;
}

const std::string& param = params[i];

Type* t = type(param);
const Type* t = type(param);
if (!t->flatten()) {
flatten(request, params, i + 1, result, callback);
return;
Expand All @@ -553,7 +602,7 @@ void MarsLanguage::flatten(const MarsRequest& request, const std::vector<std::st
}
}

void MarsLanguage::flatten(const MarsRequest& request, FlattenCallback& callback) {
void MarsLanguage::flatten(const MarsRequest& request, FlattenCallback& callback) const {
std::vector<std::string> params;
request.getParams(params);

Expand Down
Loading
Loading