Adding client for benchmark registry

README.md

@@ -92,6 +92,8 @@ qt show <run_id>
 qt compare <run_id_a> <run_id_b>
 ```
 
+When `qt run <eval_name>` does not find the requested name in the local configuration, it queries the Quantiles remote benchmark service for a benchmark that has the given name. See the [CLI's README file](./cli/README.md#remote-benchmark-fallback) for details.
+
 > Note: Pass `--json` to any of these commands to output machine- and agent-friendly JSON instead of human-formatted output.
 
 See the [CLI reference](https://quantiles.io/documentation/reference/cli) for available commands, options, and usage details.

cli/Cargo.toml

@@ -10,6 +10,8 @@ async-trait = "0.1"
 axum = "0.8.9"
 clap = { version = "4.6.1", features = ["derive"] }
 comfy-table = "7.2.1"
+buffa = "0.8"
+connectrpc = { version = "0.8", features = ["client-tls"] }
 datafusion = "46.0.0"
 dirs = "6"
 fastembed = "5"
@@ -19,6 +21,7 @@ jinja = "0.1"
 parquet = { version = "54", features = ["arrow"] }
 rand = "0.8"
 reqwest = { version = "0.13.3", features = ["json", "rustls"] }
+rustls-platform-verifier = "0.7"
 sea-orm = { version = "1.1", features = ["sqlx-sqlite", "runtime-tokio-native-tls", "time"] }
 seq-macro = "0.3"
 serde = { version = "1.0.228", features = ["derive"] }
@@ -38,6 +41,9 @@ pedantic = "deny"
 [lints.rust]
 unfulfilled_lint_expectations = "deny"
 
+[build-dependencies]
+connectrpc-build = "0.8"
+
 [dev-dependencies]
 assert_cmd = "2.1.1"
 predicates = "3.1.3"

cli/README.md

@@ -43,13 +43,22 @@ Add a `quantiles.toml` or `.quantiles.toml` file to configure an evaluation. For
 
 ```toml
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 50
 model = "openai:gpt-5.6"
 max_workers = 100
 ```
 
 See the [configuration guide](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. See the [model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up provider models, managing credentials, and troubleshooting configuration issues. Additional runnable configurations are available in [CLI configuration examples](./examples/configs) and [custom no-code examples](../custom-nocode-examples/quantiles.toml).
 
+### Remote benchmark fallback
+
+When you run `qt run <eval_name>`, the CLI first looks in the local configuration file for an evaluation called `eval_name`. If one is found, the CLI runs it immediately. If none is found, `qt` looks in 1the Quantiles remote benchmark service for an evaluation called `eval_name` and, if a match is found, downloads the benchmark definition and runs it.
+
+>If you want to override the location of the remote benchmark service, use the `--remote-url` flag or the `QUANTILES_REMOTE_URL` environment variable.
+
+When `qt` uses the remote benchmark service, downloaded remote definitions and prompt templates are verified and kept in memory for the run. They will not be cached on disk.
+
 ## Architecture
 
 The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while `qt` handles durability and observability.

cli/build.rs

@@ -0,0 +1,11 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("cargo:rerun-if-changed=proto/quantiles/benchmark/v1/benchmark_registry.proto");
+
+    connectrpc_build::Config::new()
+        .files(&["proto/quantiles/benchmark/v1/benchmark_registry.proto"])
+        .use_buf()
+        .include_file("_connectrpc.rs")
+        .compile()?;
+
+    Ok(())
+}

cli/examples/configs/anthropic/quantiles.toml

@@ -4,9 +4,12 @@
 # variable. To get an API key, go to https://platform.claude.com/settings/workspaces/default/keys
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1,000 samples. Because Anthropic charges for
-# usage, this example limits the run to 50 samples to control costs. Omit `samples` to run the
-# full benchmark.
+# usage, this configuration entry limits the run to 50 samples to control costs.
+#
+# Omit the below `samples` key to run the full benchmark.
 samples = 50
 
 # This example uses a relatively low-cost, fast model. Model details are available at:

cli/examples/configs/both_error/.quantiles.toml

@@ -1,8 +1,11 @@
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 10
 
 [benchmarks.simpleqa-verified]
+dataset = "hf://quantiles/simpleqa-verified"
 samples = 100
 
 [benchmarks.financebench]
+dataset = "hf://quantiles/financebench"
 samples = 50

cli/examples/configs/cloudflare/quantiles.toml

@@ -12,6 +12,8 @@
 # https://dash.cloudflare.com/<account_id>/ai/ai-gateway
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1,000 samples. Because Cloudflare charges for
 # usage, this example limits the run to 50 samples to control costs. Omit `samples` to run the
 # full benchmark.

cli/examples/configs/gemini/quantiles.toml

@@ -6,6 +6,8 @@
 # https://aistudio.google.com/prompts/new_chat
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1,000 samples. Because Google charges for
 # usage, this example limits the run to 50 samples to control costs. Omit `samples` to run the
 # full benchmark.

cli/examples/configs/openai/quantiles.toml

@@ -7,6 +7,8 @@
 # https://platform.openai.com/home
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1,000 samples. Because OpenAI charges for
 # usage, this example limits the run to 50 samples to control costs. Omit `samples` to run the
 # full benchmark.

cli/examples/configs/simple/quantiles.toml

@@ -8,7 +8,9 @@
 # dataset rows processed.
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 10
 
 [benchmarks.simpleqa-verified]
+dataset = "hf://quantiles/simpleqa-verified"
 samples = 100