From f5142c497d6149c8b8f26c28f9a86225c4c6222d Mon Sep 17 00:00:00 2001 From: Harry Cruz Date: Tue, 25 Aug 2026 18:23:58 +0200 Subject: [PATCH] feat: allow users to set test set names via GenerationConfig Add testSetName and testSetDescription to GenerationConfig so users can name test sets at generation time. Add TestSetClient.update() and TestSet.toBuilder() for renaming existing test sets. Rewrite stats examples to use the Insights API and update README with current code samples. Signed-off-by: Harry Cruz Co-Authored-By: Claude Opus 4.6 --- README.md | 28 +-- .../ai/rhesis/sdk/clients/TestSetClient.java | 7 + .../java/ai/rhesis/sdk/entities/TestSet.java | 2 +- .../sdk/synthesizers/ConfigSynthesizer.java | 12 +- .../sdk/synthesizers/ContextSynthesizer.java | 12 +- .../sdk/synthesizers/GenerationConfig.java | 24 +++ .../synthesizers/MultiTurnSynthesizer.java | 15 +- .../sdk/synthesizers/PromptSynthesizer.java | 12 +- .../rhesis/sdk/synthesizers/Synthesizer.java | 12 +- .../sdk/examples/GenerateTestSetExample.java | 6 + .../GenerateTestSetWithFilesExample.java | 1 + .../java/ai/rhesis/sdk/examples/README.md | 39 +++-- .../sdk/examples/TestResultStatsExample.java | 160 +++++++----------- .../sdk/examples/TestRunStatsExample.java | 100 +++++------ .../sdk/examples/TestRunWorkflowExample.java | 24 +-- .../TestSetRoundTripIntegrationTest.java | 82 +++++++++ .../sdk/unit/clients/ClientWiremockTest.java | 55 ++++++ .../synthesizers/BaseSynthesizerTest.java | 42 +++++ 18 files changed, 402 insertions(+), 231 deletions(-) diff --git a/README.md b/README.md index 4e8ab16..ee89828 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The Rhesis Java SDK empowers developers to programmatically access curated test - [Generating Custom Test Sets](#generating-custom-test-sets-%EF%B8%8F) - [Examples](#examples-) - [Test Execution](#test-execution) - - [Analytics & Stats](#analytics--stats) + - [Insights](#insights) - [About Rhesis AI](#-about-rhesis-ai) - [Community](#-community-) - [Hugging Face](#-hugging-face) @@ -65,7 +65,7 @@ The Rhesis Java SDK provides programmatic access to the Rhesis testing platform: - **Access Test Sets**: Browse and load curated test sets across multiple domains and use cases - **Generate Test Scenarios**: Create custom test sets from prompts, requirements, or domain knowledge natively using local LLM models - **Execute Test Runs**: Trigger test set runs against your endpoints in parallel or sequential mode -- **Analytics & Stats**: Typed stats for test runs and test results — pass rates by metric, requirement, category, topic, and timeline trends +- **Insights**: Aggregation queries for test runs and results — pass rates by requirement, category, topic, with date range filtering - **Manage Metrics & Tests**: Add/remove metrics on test sets, associate/disassociate tests, rescore previous runs - **Seamless Integration**: Integrate testing into your Java CI/CD pipeline and development workflow - **Comprehensive Coverage**: Scale your testing from dozens to thousands of scenarios @@ -226,8 +226,8 @@ Your API key will be in the format `rh-XXXXXXXXXXXXXXXXXXXX`. Keep this key secu ```java import ai.rhesis.sdk.RhesisClient; import ai.rhesis.sdk.entities.TestSet; -import ai.rhesis.sdk.synthesizers.Synthesizer; import ai.rhesis.sdk.synthesizers.GenerationConfig; +import ai.rhesis.sdk.synthesizers.Synthesizer; import java.util.List; public class Main { @@ -238,19 +238,27 @@ public class Main { RhesisClient client = RhesisClient.builder() .apiKey("rh-your-api-key") // Get from app.rhesis.ai settings .build(); + RhesisClient.setDefault(client); // Browse available test sets List testSets = client.testSets().list(); for (TestSet testSet : testSets) { - System.out.println(testSet.getName()); + System.out.println(testSet.name()); } - // Generate custom test scenarios - Synthesizer synthesizer = new Synthesizer("Generate tests for a medical chatbot that must never provide diagnosis"); + // Generate custom test scenarios with a name + GenerationConfig config = GenerationConfig.builder() + .generationPrompt("Generate tests for a medical chatbot that must never provide diagnosis") + .testSetName("Medical Chatbot Safety Tests") + .requirements(List.of("Refuses diagnosis", "Recommends professional consultation")) + .build(); - TestSet generatedTestSet = synthesizer.generate(10); + TestSet generatedTestSet = new Synthesizer(config).generate(10); System.out.println("Generated Tests:"); generatedTestSet.tests().forEach(test -> System.out.println(test.prompt())); + + // Push to the platform + client.testSets().create(generatedTestSet); } } ``` @@ -282,9 +290,9 @@ Looking for more detailed examples? Check out the full [Examples README](src/tes **Test Set Management** - [Test Set Metrics](src/test/java/ai/rhesis/sdk/examples/TestSetMetricsExample.java) — List, add, and remove metrics; associate and disassociate tests -**Analytics & Stats** -- [Test Run Stats](src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java) — Overall summary, status distribution, most-run test sets, timeline, filtered queries -- [Test Result Stats](src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java) — Pass rates by metric, requirement, category, and topic; per-run summaries and timeline trends +**Insights** +- [Test Run Insights](src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java) — Run counts by status, date range filtering with months and start/end dates +- [Test Result Insights](src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java) — Pass rates by requirement, category, and topic; date range queries and failed ID retrieval You can run any example from the command line using Maven. Make sure your `RHESIS_API_KEY` is set in your environment (it will be automatically picked up from a `.env` file at the root of the project if one exists): diff --git a/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java b/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java index f64f941..782aaf8 100644 --- a/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java +++ b/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java @@ -38,6 +38,13 @@ public List getTests(String id, int skip, int limit new TypeReference>() {}); } + public TestSet update(TestSet testSet) { + if (testSet.id() == null) { + throw new IllegalArgumentException("Cannot update a TestSet without an ID"); + } + return httpClient.put("/test_sets/" + testSet.id(), testSet, TestSet.class); + } + public void delete(String id) { httpClient.delete("/test_sets/" + id); } diff --git a/src/main/java/ai/rhesis/sdk/entities/TestSet.java b/src/main/java/ai/rhesis/sdk/entities/TestSet.java index 651fadd..5fd7870 100644 --- a/src/main/java/ai/rhesis/sdk/entities/TestSet.java +++ b/src/main/java/ai/rhesis/sdk/entities/TestSet.java @@ -12,7 +12,7 @@ import java.util.List; import lombok.Builder; -@Builder +@Builder(toBuilder = true) public record TestSet( @JsonProperty("id") String id, @JsonProperty("name") String name, diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java index ddd1da2..e4eed46 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java @@ -49,11 +49,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with ConfigSynthesizer", - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with ConfigSynthesizer"; + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java index 469665b..4e892e4 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java @@ -66,11 +66,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with ContextSynthesizer", - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with ContextSynthesizer"; + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java b/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java index 0203f0e..c6484da 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java @@ -9,6 +9,8 @@ public class GenerationConfig { private final List categories; private final List topics; private final String additionalContext; + private final String testSetName; + private final String testSetDescription; private GenerationConfig(Builder builder) { this.generationPrompt = builder.generationPrompt; @@ -16,6 +18,8 @@ private GenerationConfig(Builder builder) { this.categories = builder.categories != null ? builder.categories : new ArrayList<>(); this.topics = builder.topics != null ? builder.topics : new ArrayList<>(); this.additionalContext = builder.additionalContext; + this.testSetName = builder.testSetName; + this.testSetDescription = builder.testSetDescription; } public String getGenerationPrompt() { @@ -38,6 +42,14 @@ public String getAdditionalContext() { return additionalContext; } + public String getTestSetName() { + return testSetName; + } + + public String getTestSetDescription() { + return testSetDescription; + } + public static Builder builder() { return new Builder(); } @@ -48,6 +60,8 @@ public static class Builder { private List categories; private List topics; private String additionalContext; + private String testSetName; + private String testSetDescription; public Builder generationPrompt(String generationPrompt) { this.generationPrompt = generationPrompt; @@ -74,6 +88,16 @@ public Builder additionalContext(String additionalContext) { return this; } + public Builder testSetName(String testSetName) { + this.testSetName = testSetName; + return this; + } + + public Builder testSetDescription(String testSetDescription) { + this.testSetDescription = testSetDescription; + return this; + } + public GenerationConfig build() { if (generationPrompt == null) { throw new IllegalArgumentException("generationPrompt cannot be null"); diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java index ce02e31..fcb1d44 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java @@ -43,12 +43,15 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateBatch(currentBatchSize)); } - return new TestSet( - null, - "Synthesized TestSet (Multi-Turn)", - "Generated with MultiTurnSynthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.MULTI_TURN, - generatedTests); + String baseName = + config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String name = baseName + " (Multi-Turn)"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with MultiTurnSynthesizer based on prompt: " + + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.MULTI_TURN, generatedTests); } private List generateBatch(int currentBatchSize) { diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java index c4dfc06..05ac38f 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java @@ -57,11 +57,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with PromptSynthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with PromptSynthesizer based on prompt: " + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java index 8a99228..83c6331 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java @@ -53,12 +53,12 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with Synthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with Synthesizer based on prompt: " + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } public String getRenderedPrompt() { diff --git a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java index 661dab1..d17f52c 100644 --- a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java @@ -18,6 +18,7 @@ public static void main(String[] args) { GenerationConfig.builder() .generationPrompt( "You are a helpful travel assistant. You must never provide medical advice.") + .testSetName("Travel Assistant Safety Tests") .requirements(Arrays.asList("Refuses medical advice", "Provides travel itineraries")) .categories(Arrays.asList("Safety", "Functionality")) .topics(Arrays.asList("Medical", "Travel")) @@ -37,5 +38,10 @@ public static void main(String[] args) { TestSet pushedTestSet = client.testSets().create(generatedTestSet); System.out.println("Successfully pushed TestSet! ID: " + pushedTestSet.id()); + + // Rename the test set after creation + TestSet renamed = pushedTestSet.toBuilder().name("Travel Assistant Safety Tests v2").build(); + TestSet updated = client.testSets().update(renamed); + System.out.println("Renamed TestSet to: " + updated.name()); } } diff --git a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java index 68d3733..26bd640 100644 --- a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java @@ -22,6 +22,7 @@ public static void main(String[] args) throws Exception { GenerationConfig.builder() .generationPrompt( "You are testing an HR document processor. Generate tests involving reading policy documents.") + .testSetName("HR Document Processor Tests") .requirements( Arrays.asList("Accurately summarizes policies", "Identifies vacation days")) .categories(Arrays.asList("Functionality", "Document Processing")) diff --git a/src/test/java/ai/rhesis/sdk/examples/README.md b/src/test/java/ai/rhesis/sdk/examples/README.md index 5142dc0..de0a3f7 100644 --- a/src/test/java/ai/rhesis/sdk/examples/README.md +++ b/src/test/java/ai/rhesis/sdk/examples/README.md @@ -80,7 +80,7 @@ Replace the class name with any example listed below. | Example | Description | |---------|-------------| | `ExecuteTestSetExample` | Trigger a test set run against an endpoint — parallel mode, sequential mode, and with custom metrics. | -| `TestRunWorkflowExample` | Full lifecycle: list runs, inspect results, fetch stats, get last run, and rescore. | +| `TestRunWorkflowExample` | Full lifecycle: list runs, inspect results, get insights, get last run, and rescore. | ### Test Set Management @@ -88,12 +88,12 @@ Replace the class name with any example listed below. |---------|-------------| | `TestSetMetricsExample` | List, add, and remove metrics on a test set. Associate and disassociate tests. | -### Analytics & Stats +### Insights | Example | Description | |---------|-------------| -| `TestRunStatsExample` | Test run analytics: overall summary, status distribution, most-run test sets, timeline, filtering by mode/months/run IDs. | -| `TestResultStatsExample` | Test result analytics: pass rates by metric, requirement, category, and topic. Timeline trends, per-run summaries, and filtered queries. | +| `TestRunStatsExample` | Test run insights: counts by status, date range filtering with months and start/end dates. | +| `TestResultStatsExample` | Test result insights: pass rates by requirement, category, and topic. Date range queries and failed test result ID retrieval. | ## Quick Reference @@ -103,17 +103,36 @@ RhesisClient client = RhesisClient.builder() .apiKey(System.getenv("RHESIS_API_KEY")) .build(); +// Generate a named test set +GenerationConfig config = GenerationConfig.builder() + .generationPrompt("Test a customer support chatbot") + .testSetName("Support Bot Safety Tests") + .testSetDescription("Adversarial tests for the support chatbot") + .requirements(List.of("Refuses harmful requests", "Stays on topic")) + .build(); +TestSet testSet = new MultiTurnSynthesizer(config).generate(10); +client.testSets().create(testSet); + +// Rename an existing test set +TestSet existing = client.testSets().get(testSetId); +TestSet renamed = existing.toBuilder().name("New Name").build(); +client.testSets().update(renamed); + // Execute a test set Map result = client.testSets() .execute(testSetId, endpointId); -// Get test run stats -TestRunStats stats = client.testRuns().stats(); -System.out.println("Pass rate: " + stats.overallSummary().passRate() + "%"); +// Get insights (replaces stats) +InsightsResponse insights = client.insights() + .get("test_result", List.of("requirement"), List.of("count", "pass_rate"), null); -// Get test result stats by requirement -TestResultStats requirementStats = client.testResults() - .stats(TestResultStatsMode.REQUIREMENT); +// Get insights with date range +InsightsQuery query = InsightsQuery.builder("test_result") + .groupBy(List.of("category")) + .measures(List.of("count")) + .months(6) + .build(); +InsightsResponse recent = client.insights().get(query); // Get last completed run TestRun lastRun = client.testSets() diff --git a/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java b/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java index 2fbd29c..8624190 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java @@ -1,12 +1,9 @@ package ai.rhesis.sdk.examples; import ai.rhesis.sdk.RhesisClient; -import ai.rhesis.sdk.entities.TestRun; -import ai.rhesis.sdk.entities.stats.MetricStats; -import ai.rhesis.sdk.entities.stats.TestResultStats; -import ai.rhesis.sdk.entities.stats.TestRunSummary; -import ai.rhesis.sdk.entities.stats.TimelineData; -import ai.rhesis.sdk.enums.TestResultStatsMode; +import ai.rhesis.sdk.entities.InsightsIdsResponse; +import ai.rhesis.sdk.entities.InsightsQuery; +import ai.rhesis.sdk.entities.InsightsResponse; import java.util.List; import java.util.Map; @@ -14,111 +11,72 @@ public class TestResultStatsExample { public static void main(String[] args) { RhesisClient client = RhesisClient.builder().apiKey(System.getenv("RHESIS_API_KEY")).build(); - // --- Full test result stats --- - System.out.println("=== Test Result Stats (all) ==="); - TestResultStats stats = client.testResults().stats(); - - if (stats.overallPassRates() != null) { - System.out.println("Total results: " + stats.overallPassRates().total()); - System.out.println("Passed: " + stats.overallPassRates().passed()); - System.out.println("Failed: " + stats.overallPassRates().failed()); - System.out.println("Pass rate: " + stats.overallPassRates().passRate() + "%"); - } - - // --- Metric pass rates --- - if (stats.metricPassRates() != null) { - System.out.println("\n=== Metric Pass Rates ==="); - for (Map.Entry entry : stats.metricPassRates().entrySet()) { - MetricStats m = entry.getValue(); - System.out.printf( - " %-20s total=%d passed=%d failed=%d rate=%.1f%%%n", - entry.getKey(), m.total(), m.passed(), m.failed(), m.passRate()); - } - } - - // --- Requirement breakdown --- - System.out.println("\n=== Requirement Pass Rates ==="); - TestResultStats requirementStats = client.testResults().stats(TestResultStatsMode.REQUIREMENT); - if (requirementStats.requirementPassRates() != null) { - for (Map.Entry entry : - requirementStats.requirementPassRates().entrySet()) { - System.out.printf( - " %-25s rate=%.1f%% (%d/%d)%n", - entry.getKey(), - entry.getValue().passRate(), - entry.getValue().passed(), - entry.getValue().total()); - } - } - - // --- Category breakdown --- - System.out.println("\n=== Category Pass Rates ==="); - TestResultStats categoryStats = client.testResults().stats(TestResultStatsMode.CATEGORY); - if (categoryStats.categoryPassRates() != null) { - for (Map.Entry entry : categoryStats.categoryPassRates().entrySet()) { - System.out.printf(" %-25s rate=%.1f%%%n", entry.getKey(), entry.getValue().passRate()); - } + // --- Overall test result counts --- + System.out.println("=== Test Result Count ==="); + InsightsResponse overall = + client + .insights() + .get("test_result", List.of(), List.of("count", "pass_rate", "passed", "failed"), null); + for (Map row : overall.rows()) { + System.out.println(" " + row); } - // --- Topic breakdown --- - System.out.println("\n=== Topic Pass Rates ==="); - TestResultStats topicStats = client.testResults().stats(TestResultStatsMode.TOPIC); - if (topicStats.topicPassRates() != null) { - for (Map.Entry entry : topicStats.topicPassRates().entrySet()) { - System.out.printf(" %-25s rate=%.1f%%%n", entry.getKey(), entry.getValue().passRate()); - } + // --- Pass rates by requirement --- + System.out.println("\n=== Pass Rates by Requirement ==="); + InsightsResponse byRequirement = + client + .insights() + .get("test_result", List.of("requirement"), List.of("count", "pass_rate"), null); + for (Map row : byRequirement.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("requirement"), row.get("count"), row.get("pass_rate")); } - // --- Timeline --- - if (stats.timeline() != null) { - System.out.println("\n=== Timeline ==="); - for (TimelineData point : stats.timeline()) { - System.out.printf( - " %s overall: %d/%d (%.1f%%)%n", - point.date(), - point.overall().passed(), - point.overall().total(), - point.overall().passRate()); - } + // --- Pass rates by category --- + System.out.println("\n=== Pass Rates by Category ==="); + InsightsResponse byCategory = + client + .insights() + .get("test_result", List.of("category"), List.of("count", "pass_rate"), null); + for (Map row : byCategory.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("category"), row.get("count"), row.get("pass_rate")); } - // --- Per-run summary --- - if (stats.testRunSummary() != null) { - System.out.println("\n=== Per-Run Summary ==="); - for (TestRunSummary run : stats.testRunSummary()) { - System.out.printf( - " [%s] %s — %d tests, rate=%.1f%%%n", - run.id(), run.name(), run.totalTests(), run.overall().passRate()); - } + // --- Pass rates by topic --- + System.out.println("\n=== Pass Rates by Topic ==="); + InsightsResponse byTopic = + client.insights().get("test_result", List.of("topic"), List.of("count", "pass_rate"), null); + for (Map row : byTopic.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("topic"), row.get("count"), row.get("pass_rate")); } - // --- Filtered by a specific test run --- - List runs = client.testRuns().list(); - if (!runs.isEmpty()) { - System.out.println("\n=== Results for run: " + runs.get(0).name() + " ==="); - TestResultStats runStats = - client - .testResults() - .stats(TestResultStatsMode.ALL, Map.of("test_run_ids", List.of(runs.get(0).id()))); - - if (runStats.overallPassRates() != null) { - System.out.println("Pass rate: " + runStats.overallPassRates().passRate() + "%"); - } - if (runStats.metricPassRates() != null) { - System.out.println("Metrics evaluated: " + runStats.metricPassRates().size()); - } + // --- Results from the last 6 months --- + System.out.println("\n=== Test Results (last 6 months) ==="); + InsightsQuery recentQuery = + InsightsQuery.builder("test_result") + .groupBy(List.of("requirement")) + .measures(List.of("count", "pass_rate")) + .months(6) + .build(); + InsightsResponse recent = client.insights().get(recentQuery); + for (Map row : recent.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("requirement"), row.get("count"), row.get("pass_rate")); } - // --- Metadata --- - if (stats.metadata() != null) { - System.out.println("\n=== Metadata ==="); - System.out.println("Period: " + stats.metadata().period()); - System.out.println("Total runs: " + stats.metadata().totalTestRuns()); - System.out.println("Total results: " + stats.metadata().totalTestResults()); - System.out.println("Metrics: " + stats.metadata().availableMetrics()); - System.out.println("Requirements: " + stats.metadata().availableRequirements()); - System.out.println("Categories: " + stats.metadata().availableCategories()); - System.out.println("Topics: " + stats.metadata().availableTopics()); + // --- Get IDs of failed test results --- + System.out.println("\n=== Failed Test Result IDs ==="); + InsightsIdsResponse failedIds = client.insights().ids("test_result", "fail", null); + System.out.println("Entity: " + failedIds.entity()); + System.out.println("Failed IDs: " + failedIds.ids().size()); + for (String id : failedIds.ids().subList(0, Math.min(5, failedIds.ids().size()))) { + System.out.println(" " + id); } } } diff --git a/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java b/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java index 258c19b..57f2f73 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java @@ -1,13 +1,8 @@ package ai.rhesis.sdk.examples; import ai.rhesis.sdk.RhesisClient; -import ai.rhesis.sdk.entities.TestRun; -import ai.rhesis.sdk.entities.stats.StatusDistribution; -import ai.rhesis.sdk.entities.stats.TestRunStats; -import ai.rhesis.sdk.entities.stats.TestRunTimelineData; -import ai.rhesis.sdk.entities.stats.TestSetRunCount; -import ai.rhesis.sdk.enums.TestRunStatsMode; -import java.util.LinkedHashMap; +import ai.rhesis.sdk.entities.InsightsQuery; +import ai.rhesis.sdk.entities.InsightsResponse; import java.util.List; import java.util.Map; @@ -15,67 +10,48 @@ public class TestRunStatsExample { public static void main(String[] args) { RhesisClient client = RhesisClient.builder().apiKey(System.getenv("RHESIS_API_KEY")).build(); - // --- Full stats (all sections) --- - System.out.println("=== Test Run Stats (all) ==="); - TestRunStats stats = client.testRuns().stats(); - - if (stats.overallSummary() != null) { - System.out.println("Total runs: " + stats.overallSummary().totalRuns()); - System.out.println("Unique test sets: " + stats.overallSummary().uniqueTestSets()); - System.out.println("Unique executors: " + stats.overallSummary().uniqueExecutors()); - System.out.println("Pass rate: " + stats.overallSummary().passRate() + "%"); - System.out.println("Most common: " + stats.overallSummary().mostCommonStatus()); - } - - // --- Status distribution --- - if (stats.statusDistribution() != null) { - System.out.println("\n=== Status Distribution ==="); - for (StatusDistribution sd : stats.statusDistribution()) { - System.out.printf(" %-12s %d (%.1f%%)%n", sd.status(), sd.count(), sd.percentage()); - } - } - - // --- Most run test sets --- - if (stats.mostRunTestSets() != null) { - System.out.println("\n=== Most Run Test Sets ==="); - for (TestSetRunCount ts : stats.mostRunTestSets()) { - System.out.printf(" %-30s %d runs%n", ts.testSetName(), ts.runCount()); - } - } - - // --- Timeline --- - if (stats.timeline() != null) { - System.out.println("\n=== Timeline ==="); - for (TestRunTimelineData point : stats.timeline()) { - System.out.printf(" %s total_runs=%d%n", point.date(), point.totalRuns()); - } + // --- Test run count --- + System.out.println("=== Test Run Count ==="); + InsightsResponse runCount = + client.insights().get("test_run", List.of(), List.of("count"), null); + System.out.println("Dimensions: " + runCount.dimensions()); + System.out.println("Measures: " + runCount.measures()); + for (Map row : runCount.rows()) { + System.out.println(" " + row); } - // --- Summary-only mode (lighter payload) --- - System.out.println("\n=== Summary Mode ==="); - TestRunStats summary = client.testRuns().stats(TestRunStatsMode.SUMMARY); - if (summary.overallSummary() != null) { - System.out.println("Total runs: " + summary.overallSummary().totalRuns()); + // --- Test runs grouped by status --- + System.out.println("\n=== Test Runs by Status ==="); + InsightsResponse byStatus = + client.insights().get("test_run", List.of("status"), List.of("count"), null); + for (Map row : byStatus.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } - // --- Stats scoped to specific runs --- - List runs = client.testRuns().list(); - if (!runs.isEmpty()) { - System.out.println("\n=== Stats for run: " + runs.get(0).name() + " ==="); - TestRunStats scoped = client.testRuns().stats(List.of(runs.get(0).id())); - if (scoped.overallSummary() != null) { - System.out.println("Pass rate: " + scoped.overallSummary().passRate() + "%"); - } + // --- Test runs from the last 3 months --- + System.out.println("\n=== Test Runs (last 3 months) ==="); + InsightsQuery recentQuery = + InsightsQuery.builder("test_run") + .groupBy(List.of("status")) + .measures(List.of("count")) + .months(3) + .build(); + InsightsResponse recent = client.insights().get(recentQuery); + for (Map row : recent.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } - // --- Stats with custom filter params --- - System.out.println("\n=== Stats (last 3 months) ==="); - Map params = new LinkedHashMap<>(); - params.put("months", 3); - TestRunStats filtered = client.testRuns().stats(TestRunStatsMode.ALL, params); - if (filtered.metadata() != null) { - System.out.println("Period: " + filtered.metadata().period()); - System.out.println("Total runs: " + filtered.metadata().totalTestRuns()); + // --- Test runs with date range --- + System.out.println("\n=== Test Runs (custom date range) ==="); + InsightsQuery dateRangeQuery = + InsightsQuery.builder("test_run") + .measures(List.of("count")) + .startDate("2025-01-01") + .endDate("2025-12-31") + .build(); + InsightsResponse dateRange = client.insights().get(dateRangeQuery); + for (Map row : dateRange.rows()) { + System.out.println(" " + row); } } } diff --git a/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java b/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java index f8390e9..0ecd9ba 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java @@ -2,10 +2,10 @@ import ai.rhesis.sdk.RhesisClient; import ai.rhesis.sdk.entities.Endpoint; +import ai.rhesis.sdk.entities.InsightsResponse; import ai.rhesis.sdk.entities.TestResult; import ai.rhesis.sdk.entities.TestRun; import ai.rhesis.sdk.entities.TestSet; -import ai.rhesis.sdk.entities.stats.TestRunStats; import ai.rhesis.sdk.exceptions.RhesisApiException; import java.util.List; import java.util.Map; @@ -38,22 +38,12 @@ public static void main(String[] args) { } } - // --- Get test run stats --- - System.out.println("\n=== Test Run Stats ==="); - TestRunStats stats = client.testRuns().stats(); - if (stats.overallSummary() != null) { - System.out.println("Total runs: " + stats.overallSummary().totalRuns()); - System.out.println("Pass rate: " + stats.overallSummary().passRate()); - System.out.println("Unique test sets: " + stats.overallSummary().uniqueTestSets()); - } - - // Stats scoped to specific runs - if (!runs.isEmpty()) { - System.out.println("\n=== Stats for first run ==="); - TestRunStats scopedStats = client.testRuns().stats(List.of(runs.get(0).id())); - if (scopedStats.overallSummary() != null) { - System.out.println("Scoped pass rate: " + scopedStats.overallSummary().passRate()); - } + // --- Get test run insights --- + System.out.println("\n=== Test Run Insights ==="); + InsightsResponse insights = + client.insights().get("test_run", List.of("status"), List.of("count"), null); + for (Map row : insights.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } // --- Last run for a test set + endpoint pair --- diff --git a/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java b/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java index d3cc3c0..15fd7cb 100644 --- a/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java +++ b/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java @@ -145,6 +145,88 @@ void testMetadataSurvivesRoundTrip() { .containsEntry("priority", 7); } + @org.junit.jupiter.api.Test + @DisplayName("test set name can be updated via toBuilder and update") + void testSetNameCanBeUpdated() { + String suffix = UUID.randomUUID().toString().substring(0, 8); + + Test test = + Test.builder() + .requirement("Reliability") + .category("Functionality") + .topic("Naming") + .testType(TestType.SINGLE_TURN) + .prompt( + ai.rhesis.sdk.entities.Prompt.builder() + .content("Does update work?") + .expectedResponse("Yes") + .languageCode("en") + .build()) + .build(); + + TestSet toCreate = + TestSet.builder() + .name("Original Name [" + suffix + "]") + .description("Original description") + .testSetType(TestType.SINGLE_TURN) + .tests(List.of(test)) + .build(); + + TestSet created = client.testSets().create(toCreate); + createdTestSetId = created.id(); + + TestSet fetched = client.testSets().get(created.id()); + assertThat(fetched.name()).isEqualTo("Original Name [" + suffix + "]"); + + TestSet renamed = fetched.toBuilder().name("Updated Name [" + suffix + "]").build(); + TestSet updated = client.testSets().update(renamed); + + assertThat(updated.name()).isEqualTo("Updated Name [" + suffix + "]"); + assertThat(updated.description()).isEqualTo("Original description"); + + TestSet refetched = client.testSets().get(created.id()); + assertThat(refetched.name()).isEqualTo("Updated Name [" + suffix + "]"); + } + + @org.junit.jupiter.api.Test + @DisplayName("test set name and description round-trip through create and get") + void testSetNameSurvivesRoundTrip() { + String suffix = UUID.randomUUID().toString().substring(0, 8); + String name = "Custom Named TestSet [" + suffix + "]"; + String description = "Integration test for test set naming [" + suffix + "]"; + + Test test = + Test.builder() + .requirement("Reliability") + .category("Functionality") + .topic("Naming") + .testType(TestType.SINGLE_TURN) + .prompt( + ai.rhesis.sdk.entities.Prompt.builder() + .content("Does naming work?") + .expectedResponse("Yes") + .languageCode("en") + .build()) + .build(); + + TestSet toCreate = + TestSet.builder() + .name(name) + .description(description) + .testSetType(TestType.SINGLE_TURN) + .tests(List.of(test)) + .build(); + + TestSet created = client.testSets().create(toCreate); + assertThat(created).isNotNull(); + assertThat(created.id()).isNotBlank(); + createdTestSetId = created.id(); + + TestSet fetched = client.testSets().get(created.id()); + assertThat(fetched.name()).isEqualTo(name); + assertThat(fetched.description()).isEqualTo(description); + } + private static Test findByGoal(List tests, String goal) { return tests.stream() .filter(t -> t.testConfiguration() != null && goal.equals(t.testConfiguration().goal())) diff --git a/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java b/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java index 4631d26..fd53576 100644 --- a/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java +++ b/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java @@ -88,6 +88,61 @@ void testGetTestSet() { assertThat(response.testSetType()).isEqualTo(TestType.MULTI_TURN); } + @Test + void testUpdateTestSet() { + stubFor( + put(urlEqualTo("/test_sets/ts-123")) + .withHeader("Authorization", equalTo("Bearer test-key")) + .withRequestBody(matchingJsonPath("$.name", equalTo("Renamed TestSet"))) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-123\",\"name\":\"Renamed TestSet\"," + + "\"test_set_type\":\"Multi-Turn\"}"))); + + TestSet toUpdate = + TestSet.builder() + .id("ts-123") + .name("Renamed TestSet") + .testSetType(TestType.MULTI_TURN) + .build(); + TestSet response = testSetClient.update(toUpdate); + assertThat(response.id()).isEqualTo("ts-123"); + assertThat(response.name()).isEqualTo("Renamed TestSet"); + } + + @Test + void testUpdateTestSetViaToBuilder() { + stubFor( + get(urlEqualTo("/test_sets/ts-456")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-456\",\"name\":\"Old Name\"," + + "\"description\":\"Old Desc\",\"test_set_type\":\"Single-Turn\"}"))); + stubFor( + put(urlEqualTo("/test_sets/ts-456")) + .withRequestBody(matchingJsonPath("$.name", equalTo("New Name"))) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-456\",\"name\":\"New Name\"," + + "\"description\":\"Old Desc\",\"test_set_type\":\"Single-Turn\"}"))); + + TestSet existing = testSetClient.get("ts-456"); + TestSet renamed = existing.toBuilder().name("New Name").build(); + TestSet response = testSetClient.update(renamed); + + assertThat(response.name()).isEqualTo("New Name"); + assertThat(response.description()).isEqualTo("Old Desc"); + } + @Test void testGetTestRun() { stubFor( diff --git a/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java b/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java index ef24cc6..5793147 100644 --- a/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java +++ b/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java @@ -124,6 +124,48 @@ void multiTurnSynthesizerParsesNumericTurns() { assertThat(test.testConfiguration().maxTurns()).isEqualTo(5); } + @org.junit.jupiter.api.Test + void customTestSetNameIsUsedWhenProvided() { + List> flatTests = List.of(flatTest("q", "a", "en", "R", "C", "T")); + + GenerationConfig config = + GenerationConfig.builder() + .generationPrompt("test") + .testSetName("My Custom Tests") + .testSetDescription("Custom description") + .build(); + Synthesizer synth = new Synthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("My Custom Tests"); + assertThat(testSet.description()).isEqualTo("Custom description"); + } + + @org.junit.jupiter.api.Test + void defaultTestSetNameIsUsedWhenNotProvided() { + List> flatTests = List.of(flatTest("q", "a", "en", "R", "C", "T")); + + GenerationConfig config = GenerationConfig.builder().generationPrompt("test").build(); + Synthesizer synth = new Synthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("Synthesized TestSet"); + assertThat(testSet.description()).startsWith("Generated with Synthesizer"); + } + + @org.junit.jupiter.api.Test + void multiTurnSynthesizerAppendsMultiTurnSuffix() { + List> flatTests = + List.of(flatMultiTurnTest("goal", "", "", "", 2, 5, "R", "C", "T")); + + GenerationConfig config = + GenerationConfig.builder().generationPrompt("test").testSetName("Safety Suite").build(); + MultiTurnSynthesizer synth = new MultiTurnSynthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("Safety Suite (Multi-Turn)"); + } + @org.junit.jupiter.api.Test void generatedPromptHasExpectedResponseAndLanguageCodeAsTopLevelFields() throws Exception { List> flatTests =