diff --git a/README.md b/README.md index 4e8ab16..ee89828 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The Rhesis Java SDK empowers developers to programmatically access curated test - [Generating Custom Test Sets](#generating-custom-test-sets-%EF%B8%8F) - [Examples](#examples-) - [Test Execution](#test-execution) - - [Analytics & Stats](#analytics--stats) + - [Insights](#insights) - [About Rhesis AI](#-about-rhesis-ai) - [Community](#-community-) - [Hugging Face](#-hugging-face) @@ -65,7 +65,7 @@ The Rhesis Java SDK provides programmatic access to the Rhesis testing platform: - **Access Test Sets**: Browse and load curated test sets across multiple domains and use cases - **Generate Test Scenarios**: Create custom test sets from prompts, requirements, or domain knowledge natively using local LLM models - **Execute Test Runs**: Trigger test set runs against your endpoints in parallel or sequential mode -- **Analytics & Stats**: Typed stats for test runs and test results — pass rates by metric, requirement, category, topic, and timeline trends +- **Insights**: Aggregation queries for test runs and results — pass rates by requirement, category, topic, with date range filtering - **Manage Metrics & Tests**: Add/remove metrics on test sets, associate/disassociate tests, rescore previous runs - **Seamless Integration**: Integrate testing into your Java CI/CD pipeline and development workflow - **Comprehensive Coverage**: Scale your testing from dozens to thousands of scenarios @@ -226,8 +226,8 @@ Your API key will be in the format `rh-XXXXXXXXXXXXXXXXXXXX`. Keep this key secu ```java import ai.rhesis.sdk.RhesisClient; import ai.rhesis.sdk.entities.TestSet; -import ai.rhesis.sdk.synthesizers.Synthesizer; import ai.rhesis.sdk.synthesizers.GenerationConfig; +import ai.rhesis.sdk.synthesizers.Synthesizer; import java.util.List; public class Main { @@ -238,19 +238,27 @@ public class Main { RhesisClient client = RhesisClient.builder() .apiKey("rh-your-api-key") // Get from app.rhesis.ai settings .build(); + RhesisClient.setDefault(client); // Browse available test sets List testSets = client.testSets().list(); for (TestSet testSet : testSets) { - System.out.println(testSet.getName()); + System.out.println(testSet.name()); } - // Generate custom test scenarios - Synthesizer synthesizer = new Synthesizer("Generate tests for a medical chatbot that must never provide diagnosis"); + // Generate custom test scenarios with a name + GenerationConfig config = GenerationConfig.builder() + .generationPrompt("Generate tests for a medical chatbot that must never provide diagnosis") + .testSetName("Medical Chatbot Safety Tests") + .requirements(List.of("Refuses diagnosis", "Recommends professional consultation")) + .build(); - TestSet generatedTestSet = synthesizer.generate(10); + TestSet generatedTestSet = new Synthesizer(config).generate(10); System.out.println("Generated Tests:"); generatedTestSet.tests().forEach(test -> System.out.println(test.prompt())); + + // Push to the platform + client.testSets().create(generatedTestSet); } } ``` @@ -282,9 +290,9 @@ Looking for more detailed examples? Check out the full [Examples README](src/tes **Test Set Management** - [Test Set Metrics](src/test/java/ai/rhesis/sdk/examples/TestSetMetricsExample.java) — List, add, and remove metrics; associate and disassociate tests -**Analytics & Stats** -- [Test Run Stats](src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java) — Overall summary, status distribution, most-run test sets, timeline, filtered queries -- [Test Result Stats](src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java) — Pass rates by metric, requirement, category, and topic; per-run summaries and timeline trends +**Insights** +- [Test Run Insights](src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java) — Run counts by status, date range filtering with months and start/end dates +- [Test Result Insights](src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java) — Pass rates by requirement, category, and topic; date range queries and failed ID retrieval You can run any example from the command line using Maven. Make sure your `RHESIS_API_KEY` is set in your environment (it will be automatically picked up from a `.env` file at the root of the project if one exists): diff --git a/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java b/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java index f64f941..782aaf8 100644 --- a/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java +++ b/src/main/java/ai/rhesis/sdk/clients/TestSetClient.java @@ -38,6 +38,13 @@ public List getTests(String id, int skip, int limit new TypeReference>() {}); } + public TestSet update(TestSet testSet) { + if (testSet.id() == null) { + throw new IllegalArgumentException("Cannot update a TestSet without an ID"); + } + return httpClient.put("/test_sets/" + testSet.id(), testSet, TestSet.class); + } + public void delete(String id) { httpClient.delete("/test_sets/" + id); } diff --git a/src/main/java/ai/rhesis/sdk/entities/TestSet.java b/src/main/java/ai/rhesis/sdk/entities/TestSet.java index 651fadd..5fd7870 100644 --- a/src/main/java/ai/rhesis/sdk/entities/TestSet.java +++ b/src/main/java/ai/rhesis/sdk/entities/TestSet.java @@ -12,7 +12,7 @@ import java.util.List; import lombok.Builder; -@Builder +@Builder(toBuilder = true) public record TestSet( @JsonProperty("id") String id, @JsonProperty("name") String name, diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java index ddd1da2..e4eed46 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/ConfigSynthesizer.java @@ -49,11 +49,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with ConfigSynthesizer", - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with ConfigSynthesizer"; + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java index 469665b..4e892e4 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/ContextSynthesizer.java @@ -66,11 +66,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with ContextSynthesizer", - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with ContextSynthesizer"; + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java b/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java index 0203f0e..c6484da 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/GenerationConfig.java @@ -9,6 +9,8 @@ public class GenerationConfig { private final List categories; private final List topics; private final String additionalContext; + private final String testSetName; + private final String testSetDescription; private GenerationConfig(Builder builder) { this.generationPrompt = builder.generationPrompt; @@ -16,6 +18,8 @@ private GenerationConfig(Builder builder) { this.categories = builder.categories != null ? builder.categories : new ArrayList<>(); this.topics = builder.topics != null ? builder.topics : new ArrayList<>(); this.additionalContext = builder.additionalContext; + this.testSetName = builder.testSetName; + this.testSetDescription = builder.testSetDescription; } public String getGenerationPrompt() { @@ -38,6 +42,14 @@ public String getAdditionalContext() { return additionalContext; } + public String getTestSetName() { + return testSetName; + } + + public String getTestSetDescription() { + return testSetDescription; + } + public static Builder builder() { return new Builder(); } @@ -48,6 +60,8 @@ public static class Builder { private List categories; private List topics; private String additionalContext; + private String testSetName; + private String testSetDescription; public Builder generationPrompt(String generationPrompt) { this.generationPrompt = generationPrompt; @@ -74,6 +88,16 @@ public Builder additionalContext(String additionalContext) { return this; } + public Builder testSetName(String testSetName) { + this.testSetName = testSetName; + return this; + } + + public Builder testSetDescription(String testSetDescription) { + this.testSetDescription = testSetDescription; + return this; + } + public GenerationConfig build() { if (generationPrompt == null) { throw new IllegalArgumentException("generationPrompt cannot be null"); diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java index ce02e31..fcb1d44 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/MultiTurnSynthesizer.java @@ -43,12 +43,15 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateBatch(currentBatchSize)); } - return new TestSet( - null, - "Synthesized TestSet (Multi-Turn)", - "Generated with MultiTurnSynthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.MULTI_TURN, - generatedTests); + String baseName = + config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String name = baseName + " (Multi-Turn)"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with MultiTurnSynthesizer based on prompt: " + + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.MULTI_TURN, generatedTests); } private List generateBatch(int currentBatchSize) { diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java index c4dfc06..05ac38f 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/PromptSynthesizer.java @@ -57,11 +57,11 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with PromptSynthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with PromptSynthesizer based on prompt: " + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } } diff --git a/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java b/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java index 8a99228..83c6331 100644 --- a/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java +++ b/src/main/java/ai/rhesis/sdk/synthesizers/Synthesizer.java @@ -53,12 +53,12 @@ public TestSet generate(int numTests) { generatedTests.addAll(generateSingleTurnBatch(renderedPrompt)); } - return new TestSet( - null, - "Synthesized TestSet", - "Generated with Synthesizer based on prompt: " + config.getGenerationPrompt(), - TestType.SINGLE_TURN, - generatedTests); + String name = config.getTestSetName() != null ? config.getTestSetName() : "Synthesized TestSet"; + String description = + config.getTestSetDescription() != null + ? config.getTestSetDescription() + : "Generated with Synthesizer based on prompt: " + config.getGenerationPrompt(); + return new TestSet(null, name, description, TestType.SINGLE_TURN, generatedTests); } public String getRenderedPrompt() { diff --git a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java index 661dab1..d17f52c 100644 --- a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetExample.java @@ -18,6 +18,7 @@ public static void main(String[] args) { GenerationConfig.builder() .generationPrompt( "You are a helpful travel assistant. You must never provide medical advice.") + .testSetName("Travel Assistant Safety Tests") .requirements(Arrays.asList("Refuses medical advice", "Provides travel itineraries")) .categories(Arrays.asList("Safety", "Functionality")) .topics(Arrays.asList("Medical", "Travel")) @@ -37,5 +38,10 @@ public static void main(String[] args) { TestSet pushedTestSet = client.testSets().create(generatedTestSet); System.out.println("Successfully pushed TestSet! ID: " + pushedTestSet.id()); + + // Rename the test set after creation + TestSet renamed = pushedTestSet.toBuilder().name("Travel Assistant Safety Tests v2").build(); + TestSet updated = client.testSets().update(renamed); + System.out.println("Renamed TestSet to: " + updated.name()); } } diff --git a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java index 68d3733..26bd640 100644 --- a/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/GenerateTestSetWithFilesExample.java @@ -22,6 +22,7 @@ public static void main(String[] args) throws Exception { GenerationConfig.builder() .generationPrompt( "You are testing an HR document processor. Generate tests involving reading policy documents.") + .testSetName("HR Document Processor Tests") .requirements( Arrays.asList("Accurately summarizes policies", "Identifies vacation days")) .categories(Arrays.asList("Functionality", "Document Processing")) diff --git a/src/test/java/ai/rhesis/sdk/examples/README.md b/src/test/java/ai/rhesis/sdk/examples/README.md index 5142dc0..de0a3f7 100644 --- a/src/test/java/ai/rhesis/sdk/examples/README.md +++ b/src/test/java/ai/rhesis/sdk/examples/README.md @@ -80,7 +80,7 @@ Replace the class name with any example listed below. | Example | Description | |---------|-------------| | `ExecuteTestSetExample` | Trigger a test set run against an endpoint — parallel mode, sequential mode, and with custom metrics. | -| `TestRunWorkflowExample` | Full lifecycle: list runs, inspect results, fetch stats, get last run, and rescore. | +| `TestRunWorkflowExample` | Full lifecycle: list runs, inspect results, get insights, get last run, and rescore. | ### Test Set Management @@ -88,12 +88,12 @@ Replace the class name with any example listed below. |---------|-------------| | `TestSetMetricsExample` | List, add, and remove metrics on a test set. Associate and disassociate tests. | -### Analytics & Stats +### Insights | Example | Description | |---------|-------------| -| `TestRunStatsExample` | Test run analytics: overall summary, status distribution, most-run test sets, timeline, filtering by mode/months/run IDs. | -| `TestResultStatsExample` | Test result analytics: pass rates by metric, requirement, category, and topic. Timeline trends, per-run summaries, and filtered queries. | +| `TestRunStatsExample` | Test run insights: counts by status, date range filtering with months and start/end dates. | +| `TestResultStatsExample` | Test result insights: pass rates by requirement, category, and topic. Date range queries and failed test result ID retrieval. | ## Quick Reference @@ -103,17 +103,36 @@ RhesisClient client = RhesisClient.builder() .apiKey(System.getenv("RHESIS_API_KEY")) .build(); +// Generate a named test set +GenerationConfig config = GenerationConfig.builder() + .generationPrompt("Test a customer support chatbot") + .testSetName("Support Bot Safety Tests") + .testSetDescription("Adversarial tests for the support chatbot") + .requirements(List.of("Refuses harmful requests", "Stays on topic")) + .build(); +TestSet testSet = new MultiTurnSynthesizer(config).generate(10); +client.testSets().create(testSet); + +// Rename an existing test set +TestSet existing = client.testSets().get(testSetId); +TestSet renamed = existing.toBuilder().name("New Name").build(); +client.testSets().update(renamed); + // Execute a test set Map result = client.testSets() .execute(testSetId, endpointId); -// Get test run stats -TestRunStats stats = client.testRuns().stats(); -System.out.println("Pass rate: " + stats.overallSummary().passRate() + "%"); +// Get insights (replaces stats) +InsightsResponse insights = client.insights() + .get("test_result", List.of("requirement"), List.of("count", "pass_rate"), null); -// Get test result stats by requirement -TestResultStats requirementStats = client.testResults() - .stats(TestResultStatsMode.REQUIREMENT); +// Get insights with date range +InsightsQuery query = InsightsQuery.builder("test_result") + .groupBy(List.of("category")) + .measures(List.of("count")) + .months(6) + .build(); +InsightsResponse recent = client.insights().get(query); // Get last completed run TestRun lastRun = client.testSets() diff --git a/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java b/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java index 2fbd29c..8624190 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestResultStatsExample.java @@ -1,12 +1,9 @@ package ai.rhesis.sdk.examples; import ai.rhesis.sdk.RhesisClient; -import ai.rhesis.sdk.entities.TestRun; -import ai.rhesis.sdk.entities.stats.MetricStats; -import ai.rhesis.sdk.entities.stats.TestResultStats; -import ai.rhesis.sdk.entities.stats.TestRunSummary; -import ai.rhesis.sdk.entities.stats.TimelineData; -import ai.rhesis.sdk.enums.TestResultStatsMode; +import ai.rhesis.sdk.entities.InsightsIdsResponse; +import ai.rhesis.sdk.entities.InsightsQuery; +import ai.rhesis.sdk.entities.InsightsResponse; import java.util.List; import java.util.Map; @@ -14,111 +11,72 @@ public class TestResultStatsExample { public static void main(String[] args) { RhesisClient client = RhesisClient.builder().apiKey(System.getenv("RHESIS_API_KEY")).build(); - // --- Full test result stats --- - System.out.println("=== Test Result Stats (all) ==="); - TestResultStats stats = client.testResults().stats(); - - if (stats.overallPassRates() != null) { - System.out.println("Total results: " + stats.overallPassRates().total()); - System.out.println("Passed: " + stats.overallPassRates().passed()); - System.out.println("Failed: " + stats.overallPassRates().failed()); - System.out.println("Pass rate: " + stats.overallPassRates().passRate() + "%"); - } - - // --- Metric pass rates --- - if (stats.metricPassRates() != null) { - System.out.println("\n=== Metric Pass Rates ==="); - for (Map.Entry entry : stats.metricPassRates().entrySet()) { - MetricStats m = entry.getValue(); - System.out.printf( - " %-20s total=%d passed=%d failed=%d rate=%.1f%%%n", - entry.getKey(), m.total(), m.passed(), m.failed(), m.passRate()); - } - } - - // --- Requirement breakdown --- - System.out.println("\n=== Requirement Pass Rates ==="); - TestResultStats requirementStats = client.testResults().stats(TestResultStatsMode.REQUIREMENT); - if (requirementStats.requirementPassRates() != null) { - for (Map.Entry entry : - requirementStats.requirementPassRates().entrySet()) { - System.out.printf( - " %-25s rate=%.1f%% (%d/%d)%n", - entry.getKey(), - entry.getValue().passRate(), - entry.getValue().passed(), - entry.getValue().total()); - } - } - - // --- Category breakdown --- - System.out.println("\n=== Category Pass Rates ==="); - TestResultStats categoryStats = client.testResults().stats(TestResultStatsMode.CATEGORY); - if (categoryStats.categoryPassRates() != null) { - for (Map.Entry entry : categoryStats.categoryPassRates().entrySet()) { - System.out.printf(" %-25s rate=%.1f%%%n", entry.getKey(), entry.getValue().passRate()); - } + // --- Overall test result counts --- + System.out.println("=== Test Result Count ==="); + InsightsResponse overall = + client + .insights() + .get("test_result", List.of(), List.of("count", "pass_rate", "passed", "failed"), null); + for (Map row : overall.rows()) { + System.out.println(" " + row); } - // --- Topic breakdown --- - System.out.println("\n=== Topic Pass Rates ==="); - TestResultStats topicStats = client.testResults().stats(TestResultStatsMode.TOPIC); - if (topicStats.topicPassRates() != null) { - for (Map.Entry entry : topicStats.topicPassRates().entrySet()) { - System.out.printf(" %-25s rate=%.1f%%%n", entry.getKey(), entry.getValue().passRate()); - } + // --- Pass rates by requirement --- + System.out.println("\n=== Pass Rates by Requirement ==="); + InsightsResponse byRequirement = + client + .insights() + .get("test_result", List.of("requirement"), List.of("count", "pass_rate"), null); + for (Map row : byRequirement.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("requirement"), row.get("count"), row.get("pass_rate")); } - // --- Timeline --- - if (stats.timeline() != null) { - System.out.println("\n=== Timeline ==="); - for (TimelineData point : stats.timeline()) { - System.out.printf( - " %s overall: %d/%d (%.1f%%)%n", - point.date(), - point.overall().passed(), - point.overall().total(), - point.overall().passRate()); - } + // --- Pass rates by category --- + System.out.println("\n=== Pass Rates by Category ==="); + InsightsResponse byCategory = + client + .insights() + .get("test_result", List.of("category"), List.of("count", "pass_rate"), null); + for (Map row : byCategory.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("category"), row.get("count"), row.get("pass_rate")); } - // --- Per-run summary --- - if (stats.testRunSummary() != null) { - System.out.println("\n=== Per-Run Summary ==="); - for (TestRunSummary run : stats.testRunSummary()) { - System.out.printf( - " [%s] %s — %d tests, rate=%.1f%%%n", - run.id(), run.name(), run.totalTests(), run.overall().passRate()); - } + // --- Pass rates by topic --- + System.out.println("\n=== Pass Rates by Topic ==="); + InsightsResponse byTopic = + client.insights().get("test_result", List.of("topic"), List.of("count", "pass_rate"), null); + for (Map row : byTopic.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("topic"), row.get("count"), row.get("pass_rate")); } - // --- Filtered by a specific test run --- - List runs = client.testRuns().list(); - if (!runs.isEmpty()) { - System.out.println("\n=== Results for run: " + runs.get(0).name() + " ==="); - TestResultStats runStats = - client - .testResults() - .stats(TestResultStatsMode.ALL, Map.of("test_run_ids", List.of(runs.get(0).id()))); - - if (runStats.overallPassRates() != null) { - System.out.println("Pass rate: " + runStats.overallPassRates().passRate() + "%"); - } - if (runStats.metricPassRates() != null) { - System.out.println("Metrics evaluated: " + runStats.metricPassRates().size()); - } + // --- Results from the last 6 months --- + System.out.println("\n=== Test Results (last 6 months) ==="); + InsightsQuery recentQuery = + InsightsQuery.builder("test_result") + .groupBy(List.of("requirement")) + .measures(List.of("count", "pass_rate")) + .months(6) + .build(); + InsightsResponse recent = client.insights().get(recentQuery); + for (Map row : recent.rows()) { + System.out.printf( + " %-25s count=%s pass_rate=%s%n", + row.get("requirement"), row.get("count"), row.get("pass_rate")); } - // --- Metadata --- - if (stats.metadata() != null) { - System.out.println("\n=== Metadata ==="); - System.out.println("Period: " + stats.metadata().period()); - System.out.println("Total runs: " + stats.metadata().totalTestRuns()); - System.out.println("Total results: " + stats.metadata().totalTestResults()); - System.out.println("Metrics: " + stats.metadata().availableMetrics()); - System.out.println("Requirements: " + stats.metadata().availableRequirements()); - System.out.println("Categories: " + stats.metadata().availableCategories()); - System.out.println("Topics: " + stats.metadata().availableTopics()); + // --- Get IDs of failed test results --- + System.out.println("\n=== Failed Test Result IDs ==="); + InsightsIdsResponse failedIds = client.insights().ids("test_result", "fail", null); + System.out.println("Entity: " + failedIds.entity()); + System.out.println("Failed IDs: " + failedIds.ids().size()); + for (String id : failedIds.ids().subList(0, Math.min(5, failedIds.ids().size()))) { + System.out.println(" " + id); } } } diff --git a/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java b/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java index 258c19b..57f2f73 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestRunStatsExample.java @@ -1,13 +1,8 @@ package ai.rhesis.sdk.examples; import ai.rhesis.sdk.RhesisClient; -import ai.rhesis.sdk.entities.TestRun; -import ai.rhesis.sdk.entities.stats.StatusDistribution; -import ai.rhesis.sdk.entities.stats.TestRunStats; -import ai.rhesis.sdk.entities.stats.TestRunTimelineData; -import ai.rhesis.sdk.entities.stats.TestSetRunCount; -import ai.rhesis.sdk.enums.TestRunStatsMode; -import java.util.LinkedHashMap; +import ai.rhesis.sdk.entities.InsightsQuery; +import ai.rhesis.sdk.entities.InsightsResponse; import java.util.List; import java.util.Map; @@ -15,67 +10,48 @@ public class TestRunStatsExample { public static void main(String[] args) { RhesisClient client = RhesisClient.builder().apiKey(System.getenv("RHESIS_API_KEY")).build(); - // --- Full stats (all sections) --- - System.out.println("=== Test Run Stats (all) ==="); - TestRunStats stats = client.testRuns().stats(); - - if (stats.overallSummary() != null) { - System.out.println("Total runs: " + stats.overallSummary().totalRuns()); - System.out.println("Unique test sets: " + stats.overallSummary().uniqueTestSets()); - System.out.println("Unique executors: " + stats.overallSummary().uniqueExecutors()); - System.out.println("Pass rate: " + stats.overallSummary().passRate() + "%"); - System.out.println("Most common: " + stats.overallSummary().mostCommonStatus()); - } - - // --- Status distribution --- - if (stats.statusDistribution() != null) { - System.out.println("\n=== Status Distribution ==="); - for (StatusDistribution sd : stats.statusDistribution()) { - System.out.printf(" %-12s %d (%.1f%%)%n", sd.status(), sd.count(), sd.percentage()); - } - } - - // --- Most run test sets --- - if (stats.mostRunTestSets() != null) { - System.out.println("\n=== Most Run Test Sets ==="); - for (TestSetRunCount ts : stats.mostRunTestSets()) { - System.out.printf(" %-30s %d runs%n", ts.testSetName(), ts.runCount()); - } - } - - // --- Timeline --- - if (stats.timeline() != null) { - System.out.println("\n=== Timeline ==="); - for (TestRunTimelineData point : stats.timeline()) { - System.out.printf(" %s total_runs=%d%n", point.date(), point.totalRuns()); - } + // --- Test run count --- + System.out.println("=== Test Run Count ==="); + InsightsResponse runCount = + client.insights().get("test_run", List.of(), List.of("count"), null); + System.out.println("Dimensions: " + runCount.dimensions()); + System.out.println("Measures: " + runCount.measures()); + for (Map row : runCount.rows()) { + System.out.println(" " + row); } - // --- Summary-only mode (lighter payload) --- - System.out.println("\n=== Summary Mode ==="); - TestRunStats summary = client.testRuns().stats(TestRunStatsMode.SUMMARY); - if (summary.overallSummary() != null) { - System.out.println("Total runs: " + summary.overallSummary().totalRuns()); + // --- Test runs grouped by status --- + System.out.println("\n=== Test Runs by Status ==="); + InsightsResponse byStatus = + client.insights().get("test_run", List.of("status"), List.of("count"), null); + for (Map row : byStatus.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } - // --- Stats scoped to specific runs --- - List runs = client.testRuns().list(); - if (!runs.isEmpty()) { - System.out.println("\n=== Stats for run: " + runs.get(0).name() + " ==="); - TestRunStats scoped = client.testRuns().stats(List.of(runs.get(0).id())); - if (scoped.overallSummary() != null) { - System.out.println("Pass rate: " + scoped.overallSummary().passRate() + "%"); - } + // --- Test runs from the last 3 months --- + System.out.println("\n=== Test Runs (last 3 months) ==="); + InsightsQuery recentQuery = + InsightsQuery.builder("test_run") + .groupBy(List.of("status")) + .measures(List.of("count")) + .months(3) + .build(); + InsightsResponse recent = client.insights().get(recentQuery); + for (Map row : recent.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } - // --- Stats with custom filter params --- - System.out.println("\n=== Stats (last 3 months) ==="); - Map params = new LinkedHashMap<>(); - params.put("months", 3); - TestRunStats filtered = client.testRuns().stats(TestRunStatsMode.ALL, params); - if (filtered.metadata() != null) { - System.out.println("Period: " + filtered.metadata().period()); - System.out.println("Total runs: " + filtered.metadata().totalTestRuns()); + // --- Test runs with date range --- + System.out.println("\n=== Test Runs (custom date range) ==="); + InsightsQuery dateRangeQuery = + InsightsQuery.builder("test_run") + .measures(List.of("count")) + .startDate("2025-01-01") + .endDate("2025-12-31") + .build(); + InsightsResponse dateRange = client.insights().get(dateRangeQuery); + for (Map row : dateRange.rows()) { + System.out.println(" " + row); } } } diff --git a/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java b/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java index f8390e9..0ecd9ba 100644 --- a/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java +++ b/src/test/java/ai/rhesis/sdk/examples/TestRunWorkflowExample.java @@ -2,10 +2,10 @@ import ai.rhesis.sdk.RhesisClient; import ai.rhesis.sdk.entities.Endpoint; +import ai.rhesis.sdk.entities.InsightsResponse; import ai.rhesis.sdk.entities.TestResult; import ai.rhesis.sdk.entities.TestRun; import ai.rhesis.sdk.entities.TestSet; -import ai.rhesis.sdk.entities.stats.TestRunStats; import ai.rhesis.sdk.exceptions.RhesisApiException; import java.util.List; import java.util.Map; @@ -38,22 +38,12 @@ public static void main(String[] args) { } } - // --- Get test run stats --- - System.out.println("\n=== Test Run Stats ==="); - TestRunStats stats = client.testRuns().stats(); - if (stats.overallSummary() != null) { - System.out.println("Total runs: " + stats.overallSummary().totalRuns()); - System.out.println("Pass rate: " + stats.overallSummary().passRate()); - System.out.println("Unique test sets: " + stats.overallSummary().uniqueTestSets()); - } - - // Stats scoped to specific runs - if (!runs.isEmpty()) { - System.out.println("\n=== Stats for first run ==="); - TestRunStats scopedStats = client.testRuns().stats(List.of(runs.get(0).id())); - if (scopedStats.overallSummary() != null) { - System.out.println("Scoped pass rate: " + scopedStats.overallSummary().passRate()); - } + // --- Get test run insights --- + System.out.println("\n=== Test Run Insights ==="); + InsightsResponse insights = + client.insights().get("test_run", List.of("status"), List.of("count"), null); + for (Map row : insights.rows()) { + System.out.printf(" %-15s count=%s%n", row.get("status"), row.get("count")); } // --- Last run for a test set + endpoint pair --- diff --git a/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java b/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java index d3cc3c0..15fd7cb 100644 --- a/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java +++ b/src/test/java/ai/rhesis/sdk/integration/TestSetRoundTripIntegrationTest.java @@ -145,6 +145,88 @@ void testMetadataSurvivesRoundTrip() { .containsEntry("priority", 7); } + @org.junit.jupiter.api.Test + @DisplayName("test set name can be updated via toBuilder and update") + void testSetNameCanBeUpdated() { + String suffix = UUID.randomUUID().toString().substring(0, 8); + + Test test = + Test.builder() + .requirement("Reliability") + .category("Functionality") + .topic("Naming") + .testType(TestType.SINGLE_TURN) + .prompt( + ai.rhesis.sdk.entities.Prompt.builder() + .content("Does update work?") + .expectedResponse("Yes") + .languageCode("en") + .build()) + .build(); + + TestSet toCreate = + TestSet.builder() + .name("Original Name [" + suffix + "]") + .description("Original description") + .testSetType(TestType.SINGLE_TURN) + .tests(List.of(test)) + .build(); + + TestSet created = client.testSets().create(toCreate); + createdTestSetId = created.id(); + + TestSet fetched = client.testSets().get(created.id()); + assertThat(fetched.name()).isEqualTo("Original Name [" + suffix + "]"); + + TestSet renamed = fetched.toBuilder().name("Updated Name [" + suffix + "]").build(); + TestSet updated = client.testSets().update(renamed); + + assertThat(updated.name()).isEqualTo("Updated Name [" + suffix + "]"); + assertThat(updated.description()).isEqualTo("Original description"); + + TestSet refetched = client.testSets().get(created.id()); + assertThat(refetched.name()).isEqualTo("Updated Name [" + suffix + "]"); + } + + @org.junit.jupiter.api.Test + @DisplayName("test set name and description round-trip through create and get") + void testSetNameSurvivesRoundTrip() { + String suffix = UUID.randomUUID().toString().substring(0, 8); + String name = "Custom Named TestSet [" + suffix + "]"; + String description = "Integration test for test set naming [" + suffix + "]"; + + Test test = + Test.builder() + .requirement("Reliability") + .category("Functionality") + .topic("Naming") + .testType(TestType.SINGLE_TURN) + .prompt( + ai.rhesis.sdk.entities.Prompt.builder() + .content("Does naming work?") + .expectedResponse("Yes") + .languageCode("en") + .build()) + .build(); + + TestSet toCreate = + TestSet.builder() + .name(name) + .description(description) + .testSetType(TestType.SINGLE_TURN) + .tests(List.of(test)) + .build(); + + TestSet created = client.testSets().create(toCreate); + assertThat(created).isNotNull(); + assertThat(created.id()).isNotBlank(); + createdTestSetId = created.id(); + + TestSet fetched = client.testSets().get(created.id()); + assertThat(fetched.name()).isEqualTo(name); + assertThat(fetched.description()).isEqualTo(description); + } + private static Test findByGoal(List tests, String goal) { return tests.stream() .filter(t -> t.testConfiguration() != null && goal.equals(t.testConfiguration().goal())) diff --git a/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java b/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java index 4631d26..fd53576 100644 --- a/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java +++ b/src/test/java/ai/rhesis/sdk/unit/clients/ClientWiremockTest.java @@ -88,6 +88,61 @@ void testGetTestSet() { assertThat(response.testSetType()).isEqualTo(TestType.MULTI_TURN); } + @Test + void testUpdateTestSet() { + stubFor( + put(urlEqualTo("/test_sets/ts-123")) + .withHeader("Authorization", equalTo("Bearer test-key")) + .withRequestBody(matchingJsonPath("$.name", equalTo("Renamed TestSet"))) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-123\",\"name\":\"Renamed TestSet\"," + + "\"test_set_type\":\"Multi-Turn\"}"))); + + TestSet toUpdate = + TestSet.builder() + .id("ts-123") + .name("Renamed TestSet") + .testSetType(TestType.MULTI_TURN) + .build(); + TestSet response = testSetClient.update(toUpdate); + assertThat(response.id()).isEqualTo("ts-123"); + assertThat(response.name()).isEqualTo("Renamed TestSet"); + } + + @Test + void testUpdateTestSetViaToBuilder() { + stubFor( + get(urlEqualTo("/test_sets/ts-456")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-456\",\"name\":\"Old Name\"," + + "\"description\":\"Old Desc\",\"test_set_type\":\"Single-Turn\"}"))); + stubFor( + put(urlEqualTo("/test_sets/ts-456")) + .withRequestBody(matchingJsonPath("$.name", equalTo("New Name"))) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"id\":\"ts-456\",\"name\":\"New Name\"," + + "\"description\":\"Old Desc\",\"test_set_type\":\"Single-Turn\"}"))); + + TestSet existing = testSetClient.get("ts-456"); + TestSet renamed = existing.toBuilder().name("New Name").build(); + TestSet response = testSetClient.update(renamed); + + assertThat(response.name()).isEqualTo("New Name"); + assertThat(response.description()).isEqualTo("Old Desc"); + } + @Test void testGetTestRun() { stubFor( diff --git a/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java b/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java index ef24cc6..5793147 100644 --- a/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java +++ b/src/test/java/ai/rhesis/sdk/unit/synthesizers/BaseSynthesizerTest.java @@ -124,6 +124,48 @@ void multiTurnSynthesizerParsesNumericTurns() { assertThat(test.testConfiguration().maxTurns()).isEqualTo(5); } + @org.junit.jupiter.api.Test + void customTestSetNameIsUsedWhenProvided() { + List> flatTests = List.of(flatTest("q", "a", "en", "R", "C", "T")); + + GenerationConfig config = + GenerationConfig.builder() + .generationPrompt("test") + .testSetName("My Custom Tests") + .testSetDescription("Custom description") + .build(); + Synthesizer synth = new Synthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("My Custom Tests"); + assertThat(testSet.description()).isEqualTo("Custom description"); + } + + @org.junit.jupiter.api.Test + void defaultTestSetNameIsUsedWhenNotProvided() { + List> flatTests = List.of(flatTest("q", "a", "en", "R", "C", "T")); + + GenerationConfig config = GenerationConfig.builder().generationPrompt("test").build(); + Synthesizer synth = new Synthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("Synthesized TestSet"); + assertThat(testSet.description()).startsWith("Generated with Synthesizer"); + } + + @org.junit.jupiter.api.Test + void multiTurnSynthesizerAppendsMultiTurnSuffix() { + List> flatTests = + List.of(flatMultiTurnTest("goal", "", "", "", 2, 5, "R", "C", "T")); + + GenerationConfig config = + GenerationConfig.builder().generationPrompt("test").testSetName("Safety Suite").build(); + MultiTurnSynthesizer synth = new MultiTurnSynthesizer(config, stubModel(flatTests), 20); + + TestSet testSet = synth.generate(1); + assertThat(testSet.name()).isEqualTo("Safety Suite (Multi-Turn)"); + } + @org.junit.jupiter.api.Test void generatedPromptHasExpectedResponseAndLanguageCodeAsTopLevelFields() throws Exception { List> flatTests =