From 13e64c1126d5d6d58285d8967553f0e35726b29d Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 22:53:01 +0300 Subject: [PATCH 01/14] feat(bench): run explicit workload factories --- pkg/bench/metrics_provider.go | 8 ++- pkg/bench/root.go | 8 +++ pkg/bench/runtime.go | 95 ++++++++++++++++++++++++++++++----- 3 files changed, 96 insertions(+), 15 deletions(-) diff --git a/pkg/bench/metrics_provider.go b/pkg/bench/metrics_provider.go index d88b360c..bdf8aa41 100644 --- a/pkg/bench/metrics_provider.go +++ b/pkg/bench/metrics_provider.go @@ -3,6 +3,7 @@ package bench import ( "context" "fmt" + "io" "net/url" "os" "strconv" @@ -41,9 +42,12 @@ type MetricsConfig struct { // consumers (stroppy baseline) read structured numbers here. OnSummary func(metricdata.ResourceMetrics) - // Quiet suppresses the final text summary on stderr. Programmatic - // consumers that read OnSummary render their own report. + // Quiet suppresses the final text summary. Programmatic consumers that + // read OnSummary render their own report. Quiet bool + + // SummaryWriter receives the final text summary. Nil uses os.Stderr. + SummaryWriter io.Writer } func newMeterProvider( diff --git a/pkg/bench/root.go b/pkg/bench/root.go index 37b2cf3e..6d9920bc 100644 --- a/pkg/bench/root.go +++ b/pkg/bench/root.go @@ -3,7 +3,9 @@ package bench import ( "context" "errors" + "io" "net" + "os" "sync" "time" @@ -29,6 +31,7 @@ type RootState struct { metricsPrefix string onSummary func(metricdata.ResourceMetrics) quietSummary bool + summaryWriter io.Writer throughput throughput txMetrics *txMetrics @@ -62,9 +65,13 @@ func newRootState( var quiet bool + summaryWriter := io.Writer(os.Stderr) if metricsConfig != nil { onSummary = metricsConfig.OnSummary quiet = metricsConfig.Quiet + if metricsConfig.SummaryWriter != nil { + summaryWriter = metricsConfig.SummaryWriter + } } state := &RootState{ @@ -77,6 +84,7 @@ func newRootState( metricsPrefix: prefix, onSummary: onSummary, quietSummary: quiet, + summaryWriter: summaryWriter, txMetrics: &txMetrics{}, sharedSlots: make(map[uint64]*sharedDriverSlot), stepFilter: newStepFilter(steps, noSteps), diff --git a/pkg/bench/runtime.go b/pkg/bench/runtime.go index 2e088a12..215ff504 100644 --- a/pkg/bench/runtime.go +++ b/pkg/bench/runtime.go @@ -21,6 +21,9 @@ import ( "github.com/stroppy-io/stroppy/v6/pkg/report" ) +// Factory creates a fresh workload instance for one description or run. +type Factory func() Workload + // Workload is a Go-native benchmark. Define declares and binds typed parameters; // Setup runs once (schema + load steps); Iterate is the measured body driven across // VUs by the executor; Teardown runs once. @@ -65,7 +68,7 @@ func (b *Bench) Logger() *zap.Logger { return b.lg } var ( regMu sync.RWMutex - regWorkloads = map[string]func() Workload{} + regWorkloads = map[string]Factory{} errNoWorkloadRegistered = errors.New("bench: no workload registered") errDriverIndexMissing = errors.New("bench: driver index 0 not configured") @@ -80,7 +83,7 @@ var ( ) // Register adds a workload factory. Workload packages call it during init. -func Register(factory func() Workload) { +func Register(factory Factory) { if factory == nil { panic("bench: register nil workload factory") } @@ -139,13 +142,21 @@ func nilWorkload(workload Workload) bool { } } -// Describe returns a workload's deterministic parameter schema without setup or drivers. +// Describe returns a registered workload's deterministic parameter schema without setup or drivers. func Describe(name string) (Description, error) { - wl, ok := Lookup(name) + factory, ok := registeredFactory(name) if !ok { return Description{}, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) } + return DescribeFactory(factory) +} + +// DescribeFactory returns a workload factory's deterministic parameter schema without setup or drivers. +func DescribeFactory(factory Factory) (Description, error) { + wl := workloadFromFactory(factory) + name := wl.Name() + _, def, err := defineWorkload(wl, ParamInputs{}, true) if err != nil { return Description{}, fmt.Errorf("define workload %q: %w", name, err) @@ -154,6 +165,27 @@ func Describe(name string) (Description, error) { return Description{Name: name, Params: def.schema()}, nil } +func registeredFactory(name string) (Factory, bool) { + regMu.RLock() + factory, ok := regWorkloads[name] + regMu.RUnlock() + + return factory, ok +} + +func workloadFromFactory(factory Factory) Workload { + if factory == nil { + panic("bench: nil workload factory") + } + + workload := factory() + if nilWorkload(workload) || workload.Name() == "" { + panic("bench: workload factory returned an invalid workload") + } + + return workload +} + // DescribeAll returns all registered workload schemas ordered by workload name. func DescribeAll() ([]Description, error) { regMu.RLock() @@ -198,12 +230,30 @@ func Run( lg *zap.Logger, metricsConfig *MetricsConfig, ) error { - _, err := run(ctx, name, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil) + factory, ok := registeredFactory(name) + if !ok { + return fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) + } + + return RunFactory(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig) +} + +// RunFactory executes one fresh workload from factory. +func RunFactory( + ctx context.Context, + factory Factory, + drivers map[int]*config.DriverConfig, + paramInputs ParamInputs, + steps, noSteps []string, + lg *zap.Logger, + metricsConfig *MetricsConfig, +) error { + _, err := run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil) return err } -// RunWithReport executes one workload and returns its final common report even +// RunWithReport executes one registered workload and returns its final common report even // when setup, scenario, or teardown fails after report initialization. func RunWithReport( ctx context.Context, @@ -215,13 +265,34 @@ func RunWithReport( metricsConfig *MetricsConfig, reportOptions ReportOptions, ) (*report.Run, error) { - return run(ctx, name, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions) + factory, ok := registeredFactory(name) + if !ok { + return nil, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) + } + + return RunFactoryWithReport( + ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, reportOptions, + ) +} + +// RunFactoryWithReport executes one fresh workload from factory and returns its final report. +func RunFactoryWithReport( + ctx context.Context, + factory Factory, + drivers map[int]*config.DriverConfig, + paramInputs ParamInputs, + steps, noSteps []string, + lg *zap.Logger, + metricsConfig *MetricsConfig, + reportOptions ReportOptions, +) (*report.Run, error) { + return run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions) } //nolint:funlen,gocognit // lifecycle order stays explicit: setup, scenario, teardown, report. func run( ctx context.Context, - name string, + factory Factory, drivers map[int]*config.DriverConfig, paramInputs ParamInputs, steps, noSteps []string, @@ -229,10 +300,8 @@ func run( metricsConfig *MetricsConfig, reportOptions *ReportOptions, ) (*report.Run, error) { - wl, ok := Lookup(name) - if !ok { - return nil, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) - } + wl := workloadFromFactory(factory) + name := wl.Name() scenarioParams, definition, err := defineWorkload(wl, paramInputs, false) if err != nil { @@ -386,7 +455,7 @@ func finishRun( }) } - newSummary(root).printDataTo(os.Stderr, data) + newSummary(root).printDataTo(root.summaryWriter, data) return runReport, runErr } From ab68166b8e80a039740637910c3cb76147130f65 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:00:55 +0300 Subject: [PATCH 02/14] feat(runtime): inject workload catalogs --- cmd/stroppy/commands/run/run.go | 360 +++++++++++++++++--------------- pkg/bench/catalog.go | 121 +++++++++++ pkg/bench/runtime.go | 71 ++++--- 3 files changed, 355 insertions(+), 197 deletions(-) create mode 100644 pkg/bench/catalog.go diff --git a/cmd/stroppy/commands/run/run.go b/cmd/stroppy/commands/run/run.go index 8ca1e9d8..a1731073 100644 --- a/cmd/stroppy/commands/run/run.go +++ b/cmd/stroppy/commands/run/run.go @@ -67,11 +67,15 @@ var ( errReportDisabledOutput = errors.New("--no-report cannot be combined with report output options") ) -var Cmd = &cobra.Command{ - Use: "run [] [sql_file] [-f config.json] [-d driver] [-D key=value] " + - "[-e KEY=VALUE] [--steps step1,step2]", - Short: "Run a benchmark workload", - Long: `Run a Go-native benchmark workload. The first positional selects the mode: +var Cmd = NewCommand(bench.RegisteredCatalog()) + +// NewCommand builds a run command over an explicit workload catalog. +func NewCommand(catalog *bench.Catalog) *cobra.Command { + cmd := &cobra.Command{ + Use: "run [] [sql_file] [-f config.json] [-d driver] [-D key=value] " + + "[-e KEY=VALUE] [--steps step1,step2]", + Short: "Run a benchmark workload", + Long: `Run a Go-native benchmark workload. The first positional selects the mode: → registered workload stroppy run tpcc/tx .sql → SQL file stroppy run queries.sql @@ -122,10 +126,10 @@ Signals: 130 (SIGINT) or 143 (SIGTERM) after a graceful cancellation, 2 after a forced exit, and 1 for setup, validation, teardown, fatal, or other command errors. `, - DisableFlagParsing: true, - SilenceErrors: false, - ValidArgsFunction: completeRunArgs, - Example: ` + DisableFlagParsing: true, + SilenceErrors: false, + ValidArgsFunction: completeRunArgs(catalog), + Example: ` stroppy run tpcc/tx # built-in TPC-C tx workload stroppy run tpcb/tx # TPC-B tx workload stroppy run tpcb/procs # TPC-B stored-procedure variant (pg/mysql) @@ -143,165 +147,175 @@ Signals: stroppy run tpcb/tx -D driverType=csv -D url='/tmp/tpcb-csv?merge=true' \ --steps drop_schema,create_schema,load_data # dump generated rows to CSV `, - RunE: func(cmd *cobra.Command, args []string) error { - parsed, err := parseRunArgs(args) - if err != nil { - return invalidConfig(err) - } + RunE: func(cmd *cobra.Command, args []string) error { + parsed, err := parseRunArgs(args) + if err != nil { + return invalidConfig(err) + } - if parsed.help && parsed.scriptArg != "" { - return printSelectedWorkloadHelp(cmd, parsed.scriptArg, parsed.sqlArg) - } + if parsed.help && parsed.scriptArg != "" { + return printSelectedWorkloadHelp(cmd, catalog, parsed.scriptArg, parsed.sqlArg) + } - // Resolve -e values before loading configuration so logger input is ready - // before any configuration diagnostics are emitted. - envOverrides, err := runner.ResolveEnvOverrides(parsed.envArgs) - if err != nil { - return invalidConfig(err) - } + // Resolve -e values before loading configuration so logger input is ready + // before any configuration diagnostics are emitted. + envOverrides, err := runner.ResolveEnvOverrides(parsed.envArgs) + if err != nil { + return invalidConfig(err) + } - // Load configuration without emitting diagnostics. The effective logger is - // initialized immediately afterward so every following log shares it. - fileConfig, _, err := runner.LoadRunConfig(parsed.fileArg) - if err != nil { - return invalidConfig(fmt.Errorf("failed to load config file: %w", err)) - } + // Load configuration without emitting diagnostics. The effective logger is + // initialized immediately afterward so every following log shares it. + fileConfig, _, err := runner.LoadRunConfig(parsed.fileArg) + if err != nil { + return invalidConfig(fmt.Errorf("failed to load config file: %w", err)) + } - if err := initializeLogger(parsed.typedParams, envOverrides, fileConfig); err != nil { - return invalidConfig(err) - } + if err := initializeLogger(parsed.typedParams, envOverrides, fileConfig); err != nil { + return invalidConfig(err) + } - runner.LogConfigFile(fileConfig) + runner.LogConfigFile(fileConfig) - // Apply effective values: CLI overrides config file. - scriptArg := runner.EffectiveScript(parsed.scriptArg, fileConfig) - sqlArg := runner.EffectiveSQL(parsed.sqlArg, fileConfig) - steps := normalizeStepNames(runner.EffectiveSteps(parsed.steps, fileConfig)) - noSteps := normalizeStepNames(runner.EffectiveNoSteps(parsed.noSteps, fileConfig)) + // Apply effective values: CLI overrides config file. + scriptArg := runner.EffectiveScript(parsed.scriptArg, fileConfig) + sqlArg := runner.EffectiveSQL(parsed.sqlArg, fileConfig) + steps := normalizeStepNames(runner.EffectiveSteps(parsed.steps, fileConfig)) + noSteps := normalizeStepNames(runner.EffectiveNoSteps(parsed.noSteps, fileConfig)) - if parsed.help { - if scriptArg == "" { - return cmd.Help() + if parsed.help { + if scriptArg == "" { + return cmd.Help() + } + + return printSelectedWorkloadHelp(cmd, catalog, scriptArg, sqlArg) } - return printSelectedWorkloadHelp(cmd, scriptArg, sqlArg) - } + if scriptArg == "" { + return invalidConfig(errNoScript) + } - if scriptArg == "" { - return invalidConfig(errNoScript) - } + // Mutual exclusion is checked on the merged inputs (CLI over config file), + // not just CLI-vs-CLI, so `config steps + CLI --no-steps` (and vice versa) + // is rejected the same way. + if len(steps) > 0 && len(noSteps) > 0 { + return invalidConfig(errStepsMutExclusive) + } - // Mutual exclusion is checked on the merged inputs (CLI over config file), - // not just CLI-vs-CLI, so `config steps + CLI --no-steps` (and vice versa) - // is rejected the same way. - if len(steps) > 0 && len(noSteps) > 0 { - return invalidConfig(errStepsMutExclusive) - } + if len(parsed.afterDash) > 0 { + return invalidConfig(errArgsAfterDash) + } - if len(parsed.afterDash) > 0 { - return invalidConfig(errArgsAfterDash) - } + // Log override decisions when both CLI and file config are present. + if fileConfig != nil { + lg := logger.Global().Named("run") - // Log override decisions when both CLI and file config are present. - if fileConfig != nil { - lg := logger.Global().Named("run") + if parsed.scriptArg != "" && fileConfig.RunConfig.GetScript() != "" { + lg.Debug("CLI script overrides config file", + zap.String("cli", parsed.scriptArg), + zap.String("file", fileConfig.RunConfig.GetScript()), + ) + } - if parsed.scriptArg != "" && fileConfig.RunConfig.GetScript() != "" { - lg.Debug("CLI script overrides config file", - zap.String("cli", parsed.scriptArg), - zap.String("file", fileConfig.RunConfig.GetScript()), - ) + if len(parsed.steps) > 0 && len(fileConfig.RunConfig.Steps) > 0 { + lg.Debug("CLI --steps overrides config file steps", + zap.Strings("cli", parsed.steps), + zap.Strings("file", fileConfig.RunConfig.Steps), + ) + } } - if len(parsed.steps) > 0 && len(fileConfig.RunConfig.Steps) > 0 { - lg.Debug("CLI --steps overrides config file steps", - zap.Strings("cli", parsed.steps), - zap.Strings("file", fileConfig.RunConfig.Steps), - ) + paramInputs := bench.ParamInputs{ + CLI: withoutLoggerParams(parsed.typedParams), + LegacyEnv: withoutLoggerEnv(envOverrides), } - } - paramInputs := bench.ParamInputs{ - CLI: withoutLoggerParams(parsed.typedParams), - LegacyEnv: withoutLoggerEnv(envOverrides), - } - - driverConfigs := runner.DriverCLIConfigs{} + driverConfigs := runner.DriverCLIConfigs{} - if fileConfig != nil { - paramInputs.RunConfig = fileConfig.Run - paramInputs.WorkloadConfig = fileConfig.Params - paramInputs.LegacyConfigEnv = withoutLoggerEnv(fileConfig.RunConfig.Env) + if fileConfig != nil { + paramInputs.RunConfig = fileConfig.Run + paramInputs.WorkloadConfig = fileConfig.Params + paramInputs.LegacyConfigEnv = withoutLoggerEnv(fileConfig.RunConfig.Env) - driverConfigs, err = runner.DriverCLIConfigsFromFile(fileConfig.RunConfig.Drivers) - if err != nil { - return invalidConfig(err) + driverConfigs, err = runner.DriverCLIConfigsFromFile(fileConfig.RunConfig.Drivers) + if err != nil { + return invalidConfig(err) + } } - } - for idx, presetName := range parsed.driverPresets { - if err := applyDriverPreset(driverConfigs, idx, presetName); err != nil { - return invalidConfig(err) + for idx, presetName := range parsed.driverPresets { + if err := applyDriverPreset(driverConfigs, idx, presetName); err != nil { + return invalidConfig(err) + } } - } - for idx, opts := range parsed.driverOpts { - for _, kv := range opts { - if err := applyDriverOpt(driverConfigs, idx, kv[0], kv[1]); err != nil { - return invalidConfig(err) + for idx, opts := range parsed.driverOpts { + for _, kv := range opts { + if err := applyDriverOpt(driverConfigs, idx, kv[0], kv[1]); err != nil { + return invalidConfig(err) + } } } - } - // Go-native execute_sql: a .sql file, inline SQL (contains spaces), or the - // execute_sql preset routes to the Go runner with its SQL source bound as an - // explicit typed workload parameter. Checked before the registered-name - // lookup so the preset's sql arg is honored. - if name, body, file, ok := executeSQLGoRoute(scriptArg, sqlArg); ok { - return runGoWorkload( - cmd, - name, - steps, - noSteps, - withExecuteSQLSource(paramInputs, body, file), - driverConfigs, - metricsConfig(loadedRunConfig(fileConfig)), - reportOptions(loadedRunConfig(fileConfig)), - parsed.report, - ) - } + // Go-native execute_sql: a .sql file, inline SQL (contains spaces), or the + // execute_sql preset routes to the Go runner with its SQL source bound as an + // explicit typed workload parameter. Checked before the registered-name + // lookup so the preset's sql arg is honored. + if name, body, file, ok := executeSQLGoRoute(scriptArg, sqlArg); ok { + return runGoWorkload( + cmd, + catalog, + name, + steps, + noSteps, + withExecuteSQLSource(paramInputs, body, file), + driverConfigs, + metricsConfig(loadedRunConfig(fileConfig)), + reportOptions(loadedRunConfig(fileConfig)), + parsed.report, + ) + } - // Go-native workload: if a Go workload is registered under the bare - // script name, dispatch to bench.Run. - if _, ok := bench.Lookup(scriptArg); ok { - workloadParamInputs, err := withEffectiveSQLFile(scriptArg, paramInputs, sqlArg) - if err != nil { - return invalidConfig(err) + // Go-native workload: if a Go workload is registered under the bare + // script name, dispatch to bench.Run. + if _, ok := catalog.Factory(scriptArg); ok { + workloadParamInputs, err := withEffectiveSQLFile(catalog, scriptArg, paramInputs, sqlArg) + if err != nil { + return invalidConfig(err) + } + + return runGoWorkload( + cmd, + catalog, + scriptArg, + steps, + noSteps, + workloadParamInputs, + driverConfigs, + metricsConfig(loadedRunConfig(fileConfig)), + reportOptions(loadedRunConfig(fileConfig)), + parsed.report, + ) } - return runGoWorkload( - cmd, - scriptArg, - steps, - noSteps, - workloadParamInputs, - driverConfigs, - metricsConfig(loadedRunConfig(fileConfig)), - reportOptions(loadedRunConfig(fileConfig)), - parsed.report, - ) - } + return invalidConfig(fmt.Errorf("%w: %q", errUnknownWorkload, scriptArg)) + }, + } - return invalidConfig(fmt.Errorf("%w: %q", errUnknownWorkload, scriptArg)) - }, + return cmd } -func withEffectiveSQLFile(name string, inputs bench.ParamInputs, sqlFile string) (bench.ParamInputs, error) { +func withEffectiveSQLFile( + catalog *bench.Catalog, + name string, + inputs bench.ParamInputs, + sqlFile string, +) (bench.ParamInputs, error) { if sqlFile == "" { return inputs, nil } - description, err := bench.Describe(name) + description, err := catalog.Describe(name) if err != nil { return inputs, err } @@ -511,56 +525,62 @@ func metricsConfig(cfg *config.RunConfig) *bench.MetricsConfig { return metrics } -func completeRunArgs( - _ *cobra.Command, - args []string, - toComplete string, -) ([]string, cobra.ShellCompDirective) { - if !strings.HasPrefix(toComplete, "--") { - return nil, cobra.ShellCompDirectiveDefault - } +func completeRunArgs(catalog *bench.Catalog) cobra.CompletionFunc { + return func( + _ *cobra.Command, + args []string, + toComplete string, + ) ([]string, cobra.ShellCompDirective) { + if !strings.HasPrefix(toComplete, "--") { + return nil, cobra.ShellCompDirectiveDefault + } - parsed, err := parseRunArgs(args) - if err != nil || parsed.scriptArg == "" { - return nil, cobra.ShellCompDirectiveDefault - } + parsed, err := parseRunArgs(args) + if err != nil || parsed.scriptArg == "" { + return nil, cobra.ShellCompDirectiveDefault + } - describeName := parsed.scriptArg - if name, _, _, ok := executeSQLGoRoute(parsed.scriptArg, parsed.sqlArg); ok { - describeName = name - } + describeName := parsed.scriptArg + if name, _, _, ok := executeSQLGoRoute(parsed.scriptArg, parsed.sqlArg); ok { + describeName = name + } - description, err := bench.Describe(describeName) - if err != nil { - return nil, cobra.ShellCompDirectiveDefault - } + description, err := catalog.Describe(describeName) + if err != nil { + return nil, cobra.ShellCompDirectiveDefault + } - completions := make([]string, 0, len(description.Params)) - for idx := range description.Params { - param := &description.Params[idx] + completions := make([]string, 0, len(description.Params)) + for idx := range description.Params { + param := &description.Params[idx] - candidates := []string{param.Flag} - if param.Type == bench.ParamTypeBool { - candidates = []string{param.Flag + "=true", param.Flag + "=false"} - } + candidates := []string{param.Flag} + if param.Type == bench.ParamTypeBool { + candidates = []string{param.Flag + "=true", param.Flag + "=false"} + } - for _, candidate := range candidates { - if strings.HasPrefix(candidate, toComplete) { - completions = append(completions, candidate+"\t"+param.Description) + for _, candidate := range candidates { + if strings.HasPrefix(candidate, toComplete) { + completions = append(completions, candidate+"\t"+param.Description) + } } } - } - return completions, cobra.ShellCompDirectiveNoFileComp + return completions, cobra.ShellCompDirectiveNoFileComp + } } -func printSelectedWorkloadHelp(cmd *cobra.Command, scriptArg, sqlArg string) error { +func printSelectedWorkloadHelp( + cmd *cobra.Command, + catalog *bench.Catalog, + scriptArg, sqlArg string, +) error { describeName := scriptArg if name, _, _, ok := executeSQLGoRoute(scriptArg, sqlArg); ok { describeName = name } - description, err := bench.Describe(describeName) + description, err := catalog.Describe(describeName) if err != nil { return invalidConfig(fmt.Errorf("%w: %q", errUnknownWorkload, scriptArg)) } @@ -665,6 +685,7 @@ func executeSQLGoRoute(scriptArg, sqlArg string) (name, body, file string, ok bo // and step inputs are passed explicitly to their runtime owners. func runGoWorkload( cmd *cobra.Command, + catalog *bench.Catalog, name string, steps, noSteps []string, paramInputs bench.ParamInputs, @@ -695,8 +716,8 @@ func runGoWorkload( } if output.disabled { - if err := bench.Run( - cmd.Context(), name, drivers, paramInputs, steps, noSteps, logger.Global(), metrics, + if err := bench.RunCatalog( + cmd.Context(), catalog, name, drivers, paramInputs, steps, noSteps, logger.Global(), metrics, ); err != nil { return fmt.Errorf("failed to run go workload: %w", err) } @@ -704,8 +725,9 @@ func runGoWorkload( return nil } - runReport, runErr := bench.RunWithReport( - cmd.Context(), name, drivers, paramInputs, steps, noSteps, + metrics.SummaryWriter = cmd.ErrOrStderr() + runReport, runErr := bench.RunCatalogWithReport( + cmd.Context(), catalog, name, drivers, paramInputs, steps, noSteps, logger.Global(), metrics, reportConfig, ) diff --git a/pkg/bench/catalog.go b/pkg/bench/catalog.go new file mode 100644 index 00000000..fa7994e7 --- /dev/null +++ b/pkg/bench/catalog.go @@ -0,0 +1,121 @@ +package bench + +import ( + "errors" + "fmt" + "slices" +) + +var ( + errEmptyCatalog = errors.New("bench: workload catalog is empty") + errDuplicateWorkload = errors.New("bench: duplicate workload name") + errFactoryNameChanged = errors.New("bench: workload factory returned a different name") +) + +// Catalog is an explicit set of workload factories. +type Catalog struct { + factories map[string]Factory + registered bool +} + +// NewCatalog validates factories and creates an isolated workload catalog. +func NewCatalog(factories ...Factory) (*Catalog, error) { + if len(factories) == 0 { + return nil, errEmptyCatalog + } + + catalog := &Catalog{factories: make(map[string]Factory, len(factories))} + for _, factory := range factories { + workload := workloadFromFactory(factory) + name := workload.Name() + if _, exists := catalog.factories[name]; exists { + return nil, fmt.Errorf("%w %q", errDuplicateWorkload, name) + } + + catalog.factories[name] = factory + } + + return catalog, nil +} + +// RegisteredCatalog returns a live view of factories added through Register. +func RegisteredCatalog() *Catalog { + return &Catalog{registered: true} +} + +// Factory returns a workload factory by name. +func (c *Catalog) Factory(name string) (Factory, bool) { + if c == nil { + return nil, false + } + + if c.registered { + return registeredFactory(name) + } + + factory, ok := c.factories[name] + + return factory, ok +} + +// Describe returns one workload's deterministic parameter schema. +func (c *Catalog) Describe(name string) (Description, error) { + factory, ok := c.Factory(name) + if !ok { + return Description{}, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) + } + + description, err := DescribeFactory(factory) + if err != nil { + return Description{}, err + } + + if description.Name != name { + return Description{}, fmt.Errorf("%w: got %q, want %q", errFactoryNameChanged, description.Name, name) + } + + return description, nil +} + +// DescribeAll returns every workload schema ordered by workload name. +func (c *Catalog) DescribeAll() ([]Description, error) { + names := c.names() + descriptions := make([]Description, 0, len(names)) + + for _, name := range names { + description, err := c.Describe(name) + if err != nil { + return nil, err + } + + descriptions = append(descriptions, description) + } + + return descriptions, nil +} + +func (c *Catalog) names() []string { + if c == nil { + return nil + } + + if c.registered { + regMu.RLock() + names := make([]string, 0, len(regWorkloads)) + for name := range regWorkloads { + names = append(names, name) + } + regMu.RUnlock() + slices.Sort(names) + + return names + } + + names := make([]string, 0, len(c.factories)) + for name := range c.factories { + names = append(names, name) + } + slices.Sort(names) + + return names +} diff --git a/pkg/bench/runtime.go b/pkg/bench/runtime.go index 215ff504..a5ce80ae 100644 --- a/pkg/bench/runtime.go +++ b/pkg/bench/runtime.go @@ -144,12 +144,7 @@ func nilWorkload(workload Workload) bool { // Describe returns a registered workload's deterministic parameter schema without setup or drivers. func Describe(name string) (Description, error) { - factory, ok := registeredFactory(name) - if !ok { - return Description{}, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) - } - - return DescribeFactory(factory) + return RegisteredCatalog().Describe(name) } // DescribeFactory returns a workload factory's deterministic parameter schema without setup or drivers. @@ -188,28 +183,7 @@ func workloadFromFactory(factory Factory) Workload { // DescribeAll returns all registered workload schemas ordered by workload name. func DescribeAll() ([]Description, error) { - regMu.RLock() - - names := make([]string, 0, len(regWorkloads)) - for name := range regWorkloads { - names = append(names, name) - } - - regMu.RUnlock() - - slices.Sort(names) - - descriptions := make([]Description, 0, len(names)) - for _, name := range names { - description, err := Describe(name) - if err != nil { - return nil, err - } - - descriptions = append(descriptions, description) - } - - return descriptions, nil + return RegisteredCatalog().DescribeAll() } // teardownTimeout bounds workload Teardown. It runs under a detached context so @@ -275,6 +249,47 @@ func RunWithReport( ) } +// RunCatalog executes one fresh workload selected from catalog. +func RunCatalog( + ctx context.Context, + catalog *Catalog, + name string, + drivers map[int]*config.DriverConfig, + paramInputs ParamInputs, + steps, noSteps []string, + lg *zap.Logger, + metricsConfig *MetricsConfig, +) error { + factory, ok := catalog.Factory(name) + if !ok { + return fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) + } + + return RunFactory(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig) +} + +// RunCatalogWithReport executes one fresh workload selected from catalog and returns its report. +func RunCatalogWithReport( + ctx context.Context, + catalog *Catalog, + name string, + drivers map[int]*config.DriverConfig, + paramInputs ParamInputs, + steps, noSteps []string, + lg *zap.Logger, + metricsConfig *MetricsConfig, + reportOptions ReportOptions, +) (*report.Run, error) { + factory, ok := catalog.Factory(name) + if !ok { + return nil, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) + } + + return RunFactoryWithReport( + ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, reportOptions, + ) +} + // RunFactoryWithReport executes one fresh workload from factory and returns its final report. func RunFactoryWithReport( ctx context.Context, From cfd963ce8bb8635bbfc30a8800925fa384cc9927 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:03:37 +0300 Subject: [PATCH 03/14] feat(probe): describe explicit workload catalogs --- cmd/stroppy/commands/probe/probe.go | 64 +++++++++++++----------- cmd/stroppy/commands/probe/probe_test.go | 4 +- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/cmd/stroppy/commands/probe/probe.go b/cmd/stroppy/commands/probe/probe.go index 98fed322..1f4605ec 100644 --- a/cmd/stroppy/commands/probe/probe.go +++ b/cmd/stroppy/commands/probe/probe.go @@ -26,40 +26,46 @@ var ( formats = []string{humanFormat, jsonFormat} formatsWithCommas = strings.Join(formats, ", ") ErrUnsoportedFormat = errors.New("unsupported format") - Cmd = func() *cobra.Command { - cmd := &cobra.Command{ - Use: "probe", - Short: "List embedded workload presets and supported drivers", - Long: `Probe lists the embedded workload presets (their SQL dialects and docs) + Cmd = NewCommand(bench.RegisteredCatalog()) +) + +// NewCommand builds a probe command over an explicit workload catalog. +func NewCommand(workloadCatalog *bench.Catalog) *cobra.Command { + cmd := &cobra.Command{ + Use: "probe", + Short: "List embedded workload presets and supported drivers", + Long: `Probe lists the embedded workload presets (their SQL dialects and docs) and the insert methods each driver supports. Registered workload parameter schemas are read without setting up a workload or connecting to a database. -o json machine-readable output `, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - formatFlagValue := cmd.Flag(formatFlag).Value.String() - - if !contains(formats, formatFlagValue) { - return fmt.Errorf( - "%q, available (%s): %w", - formatFlagValue, - formatsWithCommas, - ErrUnsoportedFormat, - ) - } - - return printCatalog(cmd.OutOrStdout(), formatFlagValue) - }, - } + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + formatFlagValue := cmd.Flag(formatFlag).Value.String() + + if !contains(formats, formatFlagValue) { + return fmt.Errorf( + "%q, available (%s): %w", + formatFlagValue, + formatsWithCommas, + ErrUnsoportedFormat, + ) + } - cmd.Flags(). - StringP(formatFlag, string(formatFlag[0]), humanFormat, - fmt.Sprintf("(%s)", formatsWithCommas)) + return printCatalog(cmd.OutOrStdout(), workloadCatalog, formatFlagValue) + }, + } - return cmd - }() -) + cmd.Flags().StringP( + formatFlag, + string(formatFlag[0]), + humanFormat, + fmt.Sprintf("(%s)", formatsWithCommas), + ) + + return cmd +} func contains(haystack []string, needle string) bool { for _, s := range haystack { @@ -73,13 +79,13 @@ func contains(haystack []string, needle string) bool { // printCatalog renders the embedded preset catalog, workload schemas, and driver // insert-method matrix in the requested format. -func printCatalog(output io.Writer, format string) error { +func printCatalog(output io.Writer, workloadCatalog *bench.Catalog, format string) error { catalog, err := workloads.Catalog() if err != nil { return fmt.Errorf("failed to build workloads catalog: %w", err) } - descriptions, err := bench.DescribeAll() + descriptions, err := workloadCatalog.DescribeAll() if err != nil { return fmt.Errorf("failed to describe workloads: %w", err) } diff --git a/cmd/stroppy/commands/probe/probe_test.go b/cmd/stroppy/commands/probe/probe_test.go index 7df69ab3..06ad3842 100644 --- a/cmd/stroppy/commands/probe/probe_test.go +++ b/cmd/stroppy/commands/probe/probe_test.go @@ -26,7 +26,7 @@ func TestJSONCatalogIncludesWorkloadSchemas(t *testing.T) { func TestHumanCatalogIncludesGroupedWorkloads(t *testing.T) { var output bytes.Buffer - if err := printCatalog(&output, humanFormat); err != nil { + if err := printCatalog(&output, bench.RegisteredCatalog(), humanFormat); err != nil { t.Fatalf("printCatalog() error = %v", err) } @@ -50,7 +50,7 @@ func renderJSONCatalog(t *testing.T) []byte { t.Helper() var output bytes.Buffer - if err := printCatalog(&output, jsonFormat); err != nil { + if err := printCatalog(&output, bench.RegisteredCatalog(), jsonFormat); err != nil { t.Fatalf("printCatalog() error = %v", err) } From e2fc1b0099c283677b088690d339b3efdf29bb6e Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:15:59 +0300 Subject: [PATCH 04/14] feat(stroppy): expose standalone application runtime --- cmd/stroppy/commands/baseline/baseline.go | 43 +++--- cmd/stroppy/commands/help/help.go | 64 +++++---- cmd/stroppy/commands/root.go | 96 +------------ internal/cli/root.go | 157 +++++++++++++++++++++ stroppy.go | 162 ++++++++++++++++++++++ 5 files changed, 383 insertions(+), 139 deletions(-) create mode 100644 internal/cli/root.go create mode 100644 stroppy.go diff --git a/cmd/stroppy/commands/baseline/baseline.go b/cmd/stroppy/commands/baseline/baseline.go index d4a66fb3..035f5fc2 100644 --- a/cmd/stroppy/commands/baseline/baseline.go +++ b/cmd/stroppy/commands/baseline/baseline.go @@ -68,10 +68,14 @@ type options struct { var opts options // Cmd is the `stroppy baseline` subcommand. -var Cmd = &cobra.Command{ - Use: "baseline", - Short: "Measure stroppy's own performance on this machine", - Long: `Measure the stroppy ceiling on this machine: no database required. +var Cmd = NewCommand() + +// NewCommand creates an independent baseline command. +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "baseline", + Short: "Measure stroppy's own performance on this machine", + Long: `Measure the stroppy ceiling on this machine: no database required. Two tiers run the built-in baseline workload back to back: @@ -91,35 +95,36 @@ absolute thresholds, and a versioned JSON report is saved under The pg-noop server binary is resolved from an embedded copy (release builds), the ~/.stroppy/bin/ cache, or the pinned GitHub release (downloaded with consent). Use --server-path or STROPPY_PG_NOOP_PATH to supply it directly.`, - Example: ` + Example: ` stroppy baseline # full two-tier run, ~20s stroppy baseline --quick # 1s phases, smaller load stroppy baseline --tiers noop # framework tier only, no server needed stroppy baseline --json # machine-readable report on stdout stroppy baseline --server-path ./pgnoop `, - RunE: func(cmd *cobra.Command, _ []string) error { - return run(cmd.Context(), cmd.OutOrStdout()) - }, -} + RunE: func(cmd *cobra.Command, _ []string) error { + return run(cmd.Context(), cmd.OutOrStdout()) + }, + } -func init() { - Cmd.Flags().BoolVar(&opts.quick, "quick", false, "shorter phases and a smaller load") - Cmd.Flags().BoolVar(&opts.jsonOut, "json", false, "print the report as JSON") - Cmd.Flags().StringSliceVar(&opts.tiers, "tiers", []string{tierNoop, tierWire}, + cmd.Flags().BoolVar(&opts.quick, "quick", false, "shorter phases and a smaller load") + cmd.Flags().BoolVar(&opts.jsonOut, "json", false, "print the report as JSON") + cmd.Flags().StringSliceVar(&opts.tiers, "tiers", []string{tierNoop, tierWire}, "tiers to run: noop, wire") - Cmd.Flags().IntVar(&opts.vus, "vus", runtime.GOMAXPROCS(0), + cmd.Flags().IntVar(&opts.vus, "vus", runtime.GOMAXPROCS(0), "VU count for the parallel tx phase") - Cmd.Flags().DurationVar(&opts.duration, "duration", 0, + cmd.Flags().DurationVar(&opts.duration, "duration", 0, "tx phase duration (default 3s, 1s with --quick)") - Cmd.Flags().Int64Var(&opts.rows, "rows", 0, + cmd.Flags().Int64Var(&opts.rows, "rows", 0, "load rows (default 250000, 100000 with --quick)") - Cmd.Flags().StringVar(&opts.serverPath, "server-path", "", + cmd.Flags().StringVar(&opts.serverPath, "server-path", "", "path to a pg-noop binary (env STROPPY_PG_NOOP_PATH)") - Cmd.Flags().StringVar(&opts.download, "download", "ask", + cmd.Flags().StringVar(&opts.download, "download", "ask", "server download consent: ask, always, or never") - Cmd.Flags().BoolVar(&opts.noSave, "no-save", false, + cmd.Flags().BoolVar(&opts.noSave, "no-save", false, "do not write the report to ~/.stroppy/baselines/") + + return cmd } // runPlan is the validated shape of one baseline invocation. diff --git a/cmd/stroppy/commands/help/help.go b/cmd/stroppy/commands/help/help.go index 0ca21e94..b6f4692f 100644 --- a/cmd/stroppy/commands/help/help.go +++ b/cmd/stroppy/commands/help/help.go @@ -4,7 +4,7 @@ package help import ( "errors" "fmt" - "os" + "io" "strings" "github.com/spf13/cobra" @@ -28,41 +28,49 @@ func Register(t Topic) { } // Cmd is the cobra command for `stroppy help [topic]`. -var Cmd = &cobra.Command{ - Use: "help [topic]", - Short: "Show help about a topic", - Long: `Show extended help about a topic. Run without arguments to list available topics.`, - // DisableFlagParsing so that topic names like "drivers" are not mistaken for flags. - Args: cobra.ArbitraryArgs, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - printTopicList() - - return nil - } +var Cmd = NewCommand() + +// NewCommand creates an independent help command. +func NewCommand() *cobra.Command { + return &cobra.Command{ + Use: "help [topic]", + Short: "Show help about a topic", + Long: `Show extended help about a topic. Run without arguments to list available topics.`, + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return printTopicList(cmd.OutOrStdout()) + } - name := strings.ToLower(args[0]) - for _, t := range topics { - if t.Name == name { - fmt.Fprint(os.Stdout, t.Long) + name := strings.ToLower(args[0]) + for _, topic := range topics { + if topic.Name == name { + _, err := fmt.Fprint(cmd.OutOrStdout(), topic.Long) - return nil + return err + } } - } - fmt.Fprintf(os.Stderr, "stroppy help: unknown topic %q\n\n", args[0]) - printTopicList() + fmt.Fprintf(cmd.ErrOrStderr(), "stroppy help: unknown topic %q\n\n", args[0]) + _ = printTopicList(cmd.OutOrStdout()) - return fmt.Errorf("%w: %s", errUnknownTopic, args[0]) - }, + return fmt.Errorf("%w: %s", errUnknownTopic, args[0]) + }, + } } -func printTopicList() { - fmt.Fprint(os.Stdout, "Available help topics:\n\n") +func printTopicList(output io.Writer) error { + if _, err := fmt.Fprint(output, "Available help topics:\n\n"); err != nil { + return err + } - for _, t := range topics { - fmt.Fprintf(os.Stdout, " %-20s %s\n", t.Name, t.Short) + for _, topic := range topics { + if _, err := fmt.Fprintf(output, " %-20s %s\n", topic.Name, topic.Short); err != nil { + return err + } } - fmt.Fprint(os.Stdout, "\nUse 'stroppy help ' for details.\n") + _, err := fmt.Fprint(output, "\nUse 'stroppy help ' for details.\n") + + return err } diff --git a/cmd/stroppy/commands/root.go b/cmd/stroppy/commands/root.go index 8d00d46f..b25a5e09 100644 --- a/cmd/stroppy/commands/root.go +++ b/cmd/stroppy/commands/root.go @@ -2,91 +2,24 @@ package commands import ( "context" - "encoding/json" - "errors" - "fmt" - "log" "os" - "runtime/debug" "github.com/spf13/cobra" - "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/baseline" - "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/help" - "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/probe" - "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/run" - "github.com/stroppy-io/stroppy/v6/internal/version" + "github.com/stroppy-io/stroppy/v6/internal/cli" "github.com/stroppy-io/stroppy/v6/pkg/common/shutdown" _ "github.com/stroppy-io/stroppy/v6/workloads/all" ) -// appName is the binary / command name. -const appName = "stroppy" +var rootCmd = cli.NewRoot(cli.Options{IncludeBaseline: true}) -var rootCmd = &cobra.Command{ - Use: appName, - Short: "Generate and run Go-native database stress tests", -} - -// versionJSON controls whether `stroppy version` outputs machine-readable JSON. -// When more component versions are added, --json gives programmatic consumers -// a stable format to parse instead of scraping text lines. -var versionJSON bool - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Print versions of stroppy components", - Run: func(_ *cobra.Command, _ []string) { - versions := map[string]string{ - appName: version.Version, - } - - // Pull dependency versions from the compiled binary's module info. - // These stay in sync with go.mod automatically — no hardcoding. - if info, ok := debug.ReadBuildInfo(); ok { - for _, dep := range info.Deps { - if dep.Path == "github.com/jackc/pgx/v5" { - versions["pgx"] = dep.Version - } - } - } - - if versionJSON { - // Version of the native YDB SDK service-account key contract. - versions["ydb_service_account_key_file"] = "1" - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - - if err := enc.Encode(versions); err != nil { - log.Fatal(err) - } - } else { - // Fixed order for readable output. - for _, kv := range []struct{ k, v string }{ - {appName, versions[appName]}, - {"pgx", versions["pgx"]}, - } { - if kv.v != "" { - fmt.Fprintf(os.Stdout, "%-8s %s\n", kv.k, kv.v) - } - } - } - }, -} - -// Execute runs the root command under a signal-derived context and maps a -// graceful cancellation to a documented exit status: the first SIGINT/SIGTERM -// cancels the command context, and a second signal forces immediate exit. +// Execute runs the root command under a signal-derived context. func Execute() { if code := execute(); code != 0 { os.Exit(code) } } -// execute wires the cancellation context, runs Cobra, and returns the process -// exit code without terminating. Kept separate from Execute so the exit-status -// mapping is a pure function of the returned error. exitStatus is read only -// after the command returns, so it reflects whichever signal canceled the run. func execute() int { ctx, stop, exitStatus := shutdown.NotifyContext(context.Background(), nil) defer stop() @@ -96,31 +29,10 @@ func execute() int { return exitCodeFor(exitStatus(), err) } -// exitCodeFor maps a command error to a process exit status. A graceful -// cancellation uses the signal-derived code (130 SIGINT / 143 SIGTERM); any -// other error uses 1. func exitCodeFor(cancelCode int, err error) int { - if err == nil { - return 0 - } - - if errors.Is(err, context.Canceled) { - return cancelCode - } - - return 1 + return cli.ExitCodeFor(cancelCode, err) } func Root() *cobra.Command { return rootCmd } - -func init() { - cobra.EnableCommandSorting = false - rootCmd.CompletionOptions.HiddenDefaultCmd = true - - rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "%s" .Version}}`) - - versionCmd.Flags().BoolVar(&versionJSON, "json", false, "output versions as JSON") - rootCmd.AddCommand(versionCmd, run.Cmd, baseline.Cmd, probe.Cmd, help.Cmd) -} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 00000000..5d704578 --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,157 @@ +// Package cli assembles Stroppy's reusable command interface. +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "runtime/debug" + + "github.com/spf13/cobra" + + "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/baseline" + "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/help" + "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/probe" + runcommand "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/run" + "github.com/stroppy-io/stroppy/v6/internal/version" + "github.com/stroppy-io/stroppy/v6/pkg/bench" +) + +const appName = "stroppy" + +// Options selects installed or standalone command behavior. +type Options struct { + Catalog *bench.Catalog + DefaultWorkload string + IncludeBaseline bool +} + +// NewRoot creates an independent Stroppy command tree. +func NewRoot(options Options) *cobra.Command { + catalog := options.Catalog + if catalog == nil { + catalog = bench.RegisteredCatalog() + } + + root := &cobra.Command{ + Use: appName, + Short: "Generate and run Go-native database stress tests", + SilenceUsage: true, + } + root.CompletionOptions.HiddenDefaultCmd = true + root.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "%s" .Version}}`) + + run := runcommand.NewCommand(catalog) + root.AddCommand(newVersionCommand(), run, probe.NewCommand(catalog), help.NewCommand()) + if options.IncludeBaseline { + root.AddCommand(baseline.NewCommand()) + } + + if options.DefaultWorkload != "" { + root.Args = cobra.ArbitraryArgs + root.RunE = func(cmd *cobra.Command, args []string) error { + return run.RunE(run, append([]string{options.DefaultWorkload}, args...)) + } + root.SetHelpFunc(func(*cobra.Command, []string) { + _ = run.RunE(run, []string{options.DefaultWorkload, "--help"}) + }) + } + + return root +} + +// Execute runs one command tree with caller-owned context and streams. +func Execute( + ctx context.Context, + options Options, + args []string, + stdout, stderr io.Writer, +) error { + root := NewRoot(options) + root.SetArgs(args) + root.SetOut(stdout) + root.SetErr(stderr) + + return root.ExecuteContext(ctx) +} + +// ExitCodeFor maps a command error to its process exit status. +func ExitCodeFor(cancelCode int, err error) int { + if err == nil { + return 0 + } + + if errors.Is(err, context.Canceled) { + return cancelCode + } + + return 1 +} + +func newVersionCommand() *cobra.Command { + var jsonOutput bool + + cmd := &cobra.Command{ + Use: "version", + Short: "Print versions of stroppy components", + RunE: func(cmd *cobra.Command, _ []string) error { + versions := componentVersions() + if jsonOutput { + versions["ydb_service_account_key_file"] = "1" + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + + return encoder.Encode(versions) + } + + for _, item := range []struct{ name, value string }{ + {appName, versions[appName]}, + {"pgx", versions["pgx"]}, + } { + if item.value != "" { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%-8s %s\n", item.name, item.value); err != nil { + return err + } + } + } + + return nil + }, + } + cmd.Flags().BoolVar(&jsonOutput, "json", false, "output versions as JSON") + + return cmd +} + +func componentVersions() map[string]string { + versions := map[string]string{appName: version.Version} + if info, ok := debug.ReadBuildInfo(); ok { + if versions[appName] == "unknown" { + versions[appName] = moduleVersion(info) + } + + for _, dependency := range info.Deps { + if dependency.Path == "github.com/jackc/pgx/v5" { + versions["pgx"] = dependency.Version + } + } + } + + return versions +} + +func moduleVersion(info *debug.BuildInfo) string { + if info.Main.Version != "" && info.Main.Version != "(devel)" { + return info.Main.Version + } + + for _, dependency := range info.Deps { + if dependency.Path == "github.com/stroppy-io/stroppy/v6" { + return dependency.Version + } + } + + return version.Version +} diff --git a/stroppy.go b/stroppy.go new file mode 100644 index 00000000..442f2368 --- /dev/null +++ b/stroppy.go @@ -0,0 +1,162 @@ +// Package stroppy turns one workload factory into a standalone Stroppy application. +package stroppy + +import ( + "context" + "errors" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + "github.com/stroppy-io/stroppy/v6/internal/cli" + "github.com/stroppy-io/stroppy/v6/internal/version" + "github.com/stroppy-io/stroppy/v6/pkg/bench" + "github.com/stroppy-io/stroppy/v6/pkg/common/logger" + "github.com/stroppy-io/stroppy/v6/pkg/common/shutdown" + "github.com/stroppy-io/stroppy/v6/pkg/config" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/csv" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/mysql" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/noop" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/picodata" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/postgres" + _ "github.com/stroppy-io/stroppy/v6/pkg/driver/ydb" + "github.com/stroppy-io/stroppy/v6/pkg/report" +) + +// Factory creates a fresh workload for one description or run. +type Factory = bench.Factory + +// Workload is implemented by a Go-native benchmark. +type Workload = bench.Workload + +var ( + errReservedWorkloadName = errors.New("stroppy: reserved workload name") + reservedWorkloadNames = map[string]struct{}{"help": {}, "probe": {}, "version": {}} +) + +// Application is one standalone Stroppy workload. +type Application struct { + catalog *bench.Catalog + factory Factory + name string +} + +// New validates factory and creates a standalone application. +func New(factory Factory) (*Application, error) { + catalog, err := bench.NewCatalog(factory) + if err != nil { + return nil, err + } + + descriptions, err := catalog.DescribeAll() + if err != nil { + return nil, err + } + + name := descriptions[0].Name + if _, reserved := reservedWorkloadNames[name]; reserved { + return nil, fmt.Errorf("%w %q", errReservedWorkloadName, name) + } + + return &Application{catalog: catalog, factory: factory, name: name}, nil +} + +// Name returns standalone workload identity. +func (a *Application) Name() string { + if a == nil { + return "" + } + + return a.name +} + +// Execute runs standalone command behavior without exiting host process. +func (a *Application) Execute( + ctx context.Context, + args []string, + stdout, stderr io.Writer, +) error { + if a == nil { + return errors.New("stroppy: nil application") + } + + return cli.Execute(ctx, cli.Options{ + Catalog: a.catalog, DefaultWorkload: a.name, + }, args, stdout, stderr) +} + +// RunRequest contains programmatic inputs for one workload run. +type RunRequest struct { + Drivers map[int]*config.DriverConfig + Params bench.ParamInputs + Steps []string + NoSteps []string + Logger *zap.Logger + Metrics *bench.MetricsConfig + ReportOptions bench.ReportOptions +} + +// Run executes the standalone workload directly and returns its report. +func (a *Application) Run(ctx context.Context, request RunRequest) (*report.Run, error) { + if a == nil { + return nil, errors.New("stroppy: nil application") + } + + drivers := request.Drivers + if drivers == nil { + drivers = map[int]*config.DriverConfig{0: { + DriverType: config.DriverTypePostgres, + URL: "postgres://postgres:postgres@localhost:5432", + DefaultInsertMethod: "native", + }} + } + + log := request.Logger + if log == nil { + log = logger.Global() + } + + metrics := request.Metrics + if metrics == nil { + metrics = &bench.MetricsConfig{} + } + if metrics.ServiceVersion == "" { + metrics.ServiceVersion = version.Version + } + + reportOptions := request.ReportOptions + if reportOptions.StroppyVersion == "" { + reportOptions.StroppyVersion = version.Version + } + + return bench.RunFactoryWithReport( + ctx, + a.factory, + drivers, + request.Params, + request.Steps, + request.NoSteps, + log, + metrics, + reportOptions, + ) +} + +// Main runs one workload as a standalone process with Stroppy signal semantics. +func Main(factory Factory) { + application, err := New(factory) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + ctx, stop, exitStatus := shutdown.NotifyContext(context.Background(), nil) + defer stop() + + err = application.Execute(ctx, os.Args[1:], os.Stdout, os.Stderr) + if code := cli.ExitCodeFor(exitStatus(), err); code != 0 { + os.Exit(code) + } +} From 689a84b9d580adcf3947930c8a77adbaaa3942aa Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:19:25 +0300 Subject: [PATCH 05/14] refactor(cli): decouple standalone command shell --- cmd/stroppy/commands/root.go | 3 ++- internal/cli/root.go | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/stroppy/commands/root.go b/cmd/stroppy/commands/root.go index b25a5e09..18ef7902 100644 --- a/cmd/stroppy/commands/root.go +++ b/cmd/stroppy/commands/root.go @@ -6,12 +6,13 @@ import ( "github.com/spf13/cobra" + "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/baseline" "github.com/stroppy-io/stroppy/v6/internal/cli" "github.com/stroppy-io/stroppy/v6/pkg/common/shutdown" _ "github.com/stroppy-io/stroppy/v6/workloads/all" ) -var rootCmd = cli.NewRoot(cli.Options{IncludeBaseline: true}) +var rootCmd = cli.NewRoot(cli.Options{ExtraCommands: []*cobra.Command{baseline.NewCommand()}}) // Execute runs the root command under a signal-derived context. func Execute() { diff --git a/internal/cli/root.go b/internal/cli/root.go index 5d704578..3dcf0b7c 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" - "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/baseline" "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/help" "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/probe" runcommand "github.com/stroppy-io/stroppy/v6/cmd/stroppy/commands/run" @@ -25,11 +24,13 @@ const appName = "stroppy" type Options struct { Catalog *bench.Catalog DefaultWorkload string - IncludeBaseline bool + ExtraCommands []*cobra.Command } // NewRoot creates an independent Stroppy command tree. func NewRoot(options Options) *cobra.Command { + cobra.EnableCommandSorting = false + catalog := options.Catalog if catalog == nil { catalog = bench.RegisteredCatalog() @@ -45,9 +46,7 @@ func NewRoot(options Options) *cobra.Command { run := runcommand.NewCommand(catalog) root.AddCommand(newVersionCommand(), run, probe.NewCommand(catalog), help.NewCommand()) - if options.IncludeBaseline { - root.AddCommand(baseline.NewCommand()) - } + root.AddCommand(options.ExtraCommands...) if options.DefaultWorkload != "" { root.Args = cobra.ArbitraryArgs From f5a75ddb443988edaab6f62a9672a2512e9fdf8a Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:20:35 +0300 Subject: [PATCH 06/14] feat(report): save shared run history --- cmd/stroppy/commands/run/run.go | 10 +++- pkg/report/history.go | 94 +++++++++++++++++++++++++++++++++ pkg/report/history_test.go | 34 ++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 pkg/report/history.go create mode 100644 pkg/report/history_test.go diff --git a/cmd/stroppy/commands/run/run.go b/cmd/stroppy/commands/run/run.go index a1731073..61105972 100644 --- a/cmd/stroppy/commands/run/run.go +++ b/cmd/stroppy/commands/run/run.go @@ -732,8 +732,14 @@ func runGoWorkload( ) var outputErr error - if output.requested() && runReport != nil { - outputErr = writeRunReport(cmd.OutOrStdout(), output, runReport) + if runReport != nil { + if _, err := report.Save(runReport); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: save report history: %v\n", err) + } + + if output.requested() { + outputErr = writeRunReport(cmd.OutOrStdout(), output, runReport) + } } if runErr != nil { diff --git a/pkg/report/history.go b/pkg/report/history.go new file mode 100644 index 00000000..ee456ef5 --- /dev/null +++ b/pkg/report/history.go @@ -0,0 +1,94 @@ +package report + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const ( + historyDirPerm = 0o700 + historyFilePerm = 0o600 + maxNameAttempts = 1000 +) + +var errHistoryNameExhausted = errors.New("report history filename suffixes exhausted") + +// Save stores one run report under ~/.stroppy/reports and returns its path. +func Save(run *Run) (string, error) { + if run == nil { + return "", errors.New("save nil run report") + } + + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolve home dir: %w", err) + } + + dir := filepath.Join(home, ".stroppy", "reports") + if err := os.MkdirAll(dir, historyDirPerm); err != nil { + return "", fmt.Errorf("create report history: %w", err) + } + + data, err := json.MarshalIndent(run, "", " ") + if err != nil { + return "", fmt.Errorf("marshal report history: %w", err) + } + + name := run.StartedAt.UTC().Format("2006-01-02T15-04-05Z") + "-" + safeName(run.Workload) + + return writeHistoryFile(dir, name, append(data, '\n')) +} + +func writeHistoryFile(dir, name string, data []byte) (string, error) { + for suffix := 1; suffix <= maxNameAttempts; suffix++ { + fileName := name + ".json" + if suffix > 1 { + fileName = fmt.Sprintf("%s-%d.json", name, suffix) + } + + path := filepath.Join(dir, fileName) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, historyFilePerm) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return "", fmt.Errorf("create report history file: %w", err) + } + + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + removeErr := os.Remove(path) + + return "", fmt.Errorf("write report history: %w", errors.Join(writeErr, closeErr, removeErr)) + } + + return path, nil + } + + return "", fmt.Errorf("%w: %s", errHistoryNameExhausted, name) +} + +func safeName(name string) string { + var builder strings.Builder + for _, value := range strings.ToLower(name) { + switch { + case value >= 'a' && value <= 'z', value >= '0' && value <= '9', value == '-', value == '_': + builder.WriteRune(value) + default: + builder.WriteByte('-') + } + } + + name = strings.Trim(builder.String(), "-") + if name == "" { + return "workload" + } + + return name +} diff --git a/pkg/report/history_test.go b/pkg/report/history_test.go new file mode 100644 index 00000000..0faeceac --- /dev/null +++ b/pkg/report/history_test.go @@ -0,0 +1,34 @@ +package report + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestSaveWritesUniqueRunHistory(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + + run := &Run{StartedAt: time.Date(2026, time.September, 28, 12, 34, 56, 0, time.UTC), Workload: "custom/example"} + first, err := Save(run) + if err != nil { + t.Fatal(err) + } + second, err := Save(run) + if err != nil { + t.Fatal(err) + } + + if first == second { + t.Fatalf("history paths collide: %s", first) + } + + wantDir := filepath.Join(os.Getenv("HOME"), ".stroppy", "reports") + if filepath.Dir(first) != wantDir || filepath.Base(first) != "2026-09-28T12-34-56Z-custom-example.json" { + t.Fatalf("first history path = %s", first) + } + if filepath.Base(second) != "2026-09-28T12-34-56Z-custom-example-2.json" { + t.Fatalf("second history path = %s", second) + } +} From aecfa03fc05ae1ef01aac65560fd5dba4c8197d4 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:28:30 +0300 Subject: [PATCH 07/14] test(stroppy): cover standalone commands --- cmd/stroppy/commands/run/run.go | 12 +++-- internal/cli/root.go | 28 ++++++---- stroppy_test.go | 95 +++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 stroppy_test.go diff --git a/cmd/stroppy/commands/run/run.go b/cmd/stroppy/commands/run/run.go index 61105972..28bc195b 100644 --- a/cmd/stroppy/commands/run/run.go +++ b/cmd/stroppy/commands/run/run.go @@ -67,10 +67,11 @@ var ( errReportDisabledOutput = errors.New("--no-report cannot be combined with report output options") ) -var Cmd = NewCommand(bench.RegisteredCatalog()) +var Cmd = NewCommand(bench.RegisteredCatalog(), "") -// NewCommand builds a run command over an explicit workload catalog. -func NewCommand(catalog *bench.Catalog) *cobra.Command { +// NewCommand builds a run command over an explicit workload catalog. A non-empty +// default workload accepts flags without a workload positional. +func NewCommand(catalog *bench.Catalog, defaultWorkload string) *cobra.Command { cmd := &cobra.Command{ Use: "run [] [sql_file] [-f config.json] [-d driver] [-D key=value] " + "[-e KEY=VALUE] [--steps step1,step2]", @@ -148,6 +149,10 @@ Signals: --steps drop_schema,create_schema,load_data # dump generated rows to CSV `, RunE: func(cmd *cobra.Command, args []string) error { + if defaultWorkload != "" { + args = append([]string{defaultWorkload}, args...) + } + parsed, err := parseRunArgs(args) if err != nil { return invalidConfig(err) @@ -716,6 +721,7 @@ func runGoWorkload( } if output.disabled { + metrics.SummaryWriter = cmd.ErrOrStderr() if err := bench.RunCatalog( cmd.Context(), catalog, name, drivers, paramInputs, steps, noSteps, logger.Global(), metrics, ); err != nil { diff --git a/internal/cli/root.go b/internal/cli/root.go index 3dcf0b7c..bb6b9a41 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -37,24 +37,34 @@ func NewRoot(options Options) *cobra.Command { } root := &cobra.Command{ - Use: appName, - Short: "Generate and run Go-native database stress tests", - SilenceUsage: true, + Use: appName, + Short: "Generate and run Go-native database stress tests", + SilenceUsage: true, + DisableFlagParsing: options.DefaultWorkload != "", } root.CompletionOptions.HiddenDefaultCmd = true root.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "%s" .Version}}`) - run := runcommand.NewCommand(catalog) - root.AddCommand(newVersionCommand(), run, probe.NewCommand(catalog), help.NewCommand()) + run := runcommand.NewCommand(catalog, options.DefaultWorkload) + root.AddCommand(newVersionCommand(), probe.NewCommand(catalog), help.NewCommand()) root.AddCommand(options.ExtraCommands...) - if options.DefaultWorkload != "" { + if options.DefaultWorkload == "" { + root.AddCommand(run) + } else { root.Args = cobra.ArbitraryArgs root.RunE = func(cmd *cobra.Command, args []string) error { - return run.RunE(run, append([]string{options.DefaultWorkload}, args...)) + run.SetContext(cmd.Context()) + run.SetOut(cmd.OutOrStdout()) + run.SetErr(cmd.ErrOrStderr()) + + return run.RunE(run, args) } - root.SetHelpFunc(func(*cobra.Command, []string) { - _ = run.RunE(run, []string{options.DefaultWorkload, "--help"}) + root.SetHelpFunc(func(cmd *cobra.Command, _ []string) { + run.SetContext(cmd.Context()) + run.SetOut(cmd.OutOrStdout()) + run.SetErr(cmd.ErrOrStderr()) + _ = run.RunE(run, []string{"--help"}) }) } diff --git a/stroppy_test.go b/stroppy_test.go new file mode 100644 index 00000000..1fd8959f --- /dev/null +++ b/stroppy_test.go @@ -0,0 +1,95 @@ +package stroppy_test + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + stroppy "github.com/stroppy-io/stroppy/v6" + "github.com/stroppy-io/stroppy/v6/pkg/bench" + "github.com/stroppy-io/stroppy/v6/pkg/config" +) + +type namedWorkload struct{ name string } + +func (workload *namedWorkload) Name() string { return workload.name } +func (*namedWorkload) Define(*bench.Def) error { return nil } +func (*namedWorkload) Setup(context.Context, *bench.Bench) error { return nil } +func (*namedWorkload) Iterate(context.Context, *bench.Bench) error { return nil } +func (*namedWorkload) Teardown(context.Context, *bench.Bench) error { return nil } + +func newWorkload() bench.Workload { return &namedWorkload{name: "external/example"} } + +func TestNewRejectsReservedWorkloadName(t *testing.T) { + _, err := stroppy.New(func() bench.Workload { return &namedWorkload{name: "probe"} }) + if err == nil || !strings.Contains(err.Error(), "reserved workload name") { + t.Fatalf("New() error = %v", err) + } +} + +func TestApplicationRun(t *testing.T) { + app, err := stroppy.New(newWorkload) + if err != nil { + t.Fatal(err) + } + + report, err := app.Run(t.Context(), stroppy.RunRequest{ + Drivers: map[int]*config.DriverConfig{0: {DriverType: config.DriverTypeNoop}}, + Params: bench.ParamInputs{CLI: map[string]string{"iterations": "2"}}, + Metrics: &bench.MetricsConfig{Quiet: true}, + }) + if err != nil { + t.Fatal(err) + } + + if report.Workload != "external/example" || report.Driver != "noop" { + t.Fatalf("report identity = %s/%s", report.Workload, report.Driver) + } +} + +func TestApplicationExecute(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + + app, err := stroppy.New(newWorkload) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + args []string + want string + }{ + {"help", []string{"--help"}, "external/example"}, + {"probe", []string{"probe", "-o", "json"}, `"external/example"`}, + {"version", []string{"version", "--json"}, `"stroppy"`}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + if err := app.Execute(t.Context(), test.args, &stdout, &stderr); err != nil { + t.Fatalf("Execute() error = %v, stderr = %q", err, stderr.String()) + } + if !strings.Contains(stdout.String(), test.want) { + t.Fatalf("stdout = %q, want %q", stdout.String(), test.want) + } + }) + } + + var stdout, stderr bytes.Buffer + err = app.Execute(t.Context(), []string{ + "-d", "noop", "--iterations", "2", "--report-format", "json", + }, &stdout, &stderr) + if err != nil { + t.Fatalf("default run error = %v, stderr = %q", err, stderr.String()) + } + + var report map[string]any + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode report: %v\n%s", err, stdout.String()) + } + if report["workload"] != "external/example" { + t.Fatalf("workload = %v", report["workload"]) + } +} From 162d7d835eacd260d0f6bb9f0fcac8a479f13ee3 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:38:25 +0300 Subject: [PATCH 08/14] test(module): run external standalone workload --- test/modulecompat/module_test.go | 152 +++++++++++++++--- .../testdata/consumer/main.go.txt | 17 +- 2 files changed, 149 insertions(+), 20 deletions(-) diff --git a/test/modulecompat/module_test.go b/test/modulecompat/module_test.go index a01e7c75..4da64fc5 100644 --- a/test/modulecompat/module_test.go +++ b/test/modulecompat/module_test.go @@ -2,11 +2,14 @@ package modulecompat_test import ( "archive/zip" + "errors" "fmt" + "io/fs" "net/url" "os" "os/exec" "path/filepath" + "strings" "testing" ) @@ -16,23 +19,74 @@ const ( ) func TestExternalModuleResolvesV6Release(t *testing.T) { - repoRoot := filepath.Clean(filepath.Join("..", "..")) + repoRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } proxyDir := t.TempDir() writeModuleProxy(t, proxyDir, repoRoot) - consumerDir := t.TempDir() - copyFile(t, filepath.Join("testdata", "consumer", "go.mod"), filepath.Join(consumerDir, "go.mod")) - copyFile(t, filepath.Join("testdata", "consumer", "main.go.txt"), filepath.Join(consumerDir, "main.go")) - + consumerDir := prepareConsumer(t) + binary := filepath.Join(consumerDir, "consumer") proxyURL := (&url.URL{Scheme: "file", Path: proxyDir}).String() - cmd := exec.Command("go", "build", "-mod=mod", "-o", filepath.Join(consumerDir, "consumer"), ".") - cmd.Dir = consumerDir + moduleCache := filepath.Join(t.TempDir(), "modcache") + t.Cleanup(func() { makeWritable(t, moduleCache) }) + buildEnv := append(os.Environ(), + "GOPROXY="+proxyURL+",https://proxy.golang.org,direct", + "GONOSUMDB="+modulePath, + "GOWORK=off", + "GOMODCACHE="+moduleCache, + "GOCACHE="+filepath.Join(t.TempDir(), "gocache"), + ) + + runCommand(t, consumerDir, buildEnv, "go", "build", "-mod=mod", "-o", binary, ".") + + home := t.TempDir() + runEnv := append(os.Environ(), "HOME="+home) + for _, test := range []struct { + args []string + want string + }{ + {[]string{"--help"}, "external/fixture"}, + {[]string{"probe", "-o", "json"}, `"external/fixture"`}, + {[]string{"version", "--json"}, `"stroppy"`}, + {[]string{"-d", "noop", "--iterations", "2", "--report-format", "json"}, `"workload": "external/fixture"`}, + } { + output := runCommand(t, consumerDir, runEnv, binary, test.args...) + if !strings.Contains(output, test.want) { + t.Fatalf("%s output = %q, want %q", test.args, output, test.want) + } + } - cmd.Env = append(os.Environ(), "GOPROXY="+proxyURL+",off", "GOSUMDB=off", "GOWORK=off") + history, err := filepath.Glob(filepath.Join(home, ".stroppy", "reports", "*.json")) + if err != nil || len(history) != 1 { + t.Fatalf("report history = %v, error = %v", history, err) + } +} + +func prepareConsumer(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + copyFile(t, filepath.Join("testdata", "consumer", "go.mod"), filepath.Join(dir, "go.mod")) + copyFile(t, filepath.Join("testdata", "consumer", "main.go.txt"), filepath.Join(dir, "main.go")) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("build external consumer: %v\n%s", err, output) + return dir +} + +func runCommand(t *testing.T, dir string, env []string, name string, args ...string) string { + t.Helper() + + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Env = env + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s %v: %v\n%s", name, args, err, output) } + + return string(output) } func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { @@ -59,20 +113,46 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { archive := zip.NewWriter(file) prefix := modulePath + "@" + version + "/" - for _, name := range []string{"go.mod", "pkg/report/report.go"} { - body, err := os.ReadFile(filepath.Join(repoRoot, filepath.FromSlash(name))) - if err != nil { - t.Fatal(err) + archived := 0 + err = filepath.WalkDir(repoRoot, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + name := entry.Name() + if path != repoRoot && (strings.HasPrefix(name, ".") || name == "build" || name == "bin") { + return filepath.SkipDir + } + + return nil } - entry, err := archive.Create(prefix + name) + relative, err := filepath.Rel(repoRoot, path) if err != nil { - t.Fatal(err) + return err + } + if !moduleArchiveFile(relative) { + return nil } - if _, err := entry.Write(body); err != nil { - t.Fatal(err) + body, err := os.ReadFile(path) + if err != nil { + return err } + writer, err := archive.Create(prefix + filepath.ToSlash(relative)) + if err != nil { + return err + } + _, err = writer.Write(body) + archived++ + + return err + }) + if err != nil { + t.Fatal(err) + } + if archived == 0 { + t.Fatal("module archive is empty") } if err := archive.Close(); err != nil { @@ -84,6 +164,42 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { } } +func moduleArchiveFile(path string) bool { + if path == "go.mod" || path == "go.sum" || path == "LICENSE" { + return true + } + + if strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") { + return true + } + + for _, prefix := range []string{"third_party/", "workloads/"} { + if strings.HasPrefix(filepath.ToSlash(path), prefix) { + return true + } + } + + return false +} + +func makeWritable(t *testing.T, root string) { + t.Helper() + + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return nil + } + if entry.IsDir() { + return os.Chmod(path, 0o700) + } + + return os.Chmod(path, 0o600) + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + t.Errorf("make module cache writable: %v", err) + } +} + func copyFile(t *testing.T, source, target string) { t.Helper() diff --git a/test/modulecompat/testdata/consumer/main.go.txt b/test/modulecompat/testdata/consumer/main.go.txt index 11b26240..34534109 100644 --- a/test/modulecompat/testdata/consumer/main.go.txt +++ b/test/modulecompat/testdata/consumer/main.go.txt @@ -1,7 +1,20 @@ package main -import "github.com/stroppy-io/stroppy/v6/pkg/report" +import ( + "context" + + stroppy "github.com/stroppy-io/stroppy/v6" + "github.com/stroppy-io/stroppy/v6/pkg/bench" +) + +type workload struct{} + +func (*workload) Name() string { return "external/fixture" } +func (*workload) Define(*bench.Def) error { return nil } +func (*workload) Setup(context.Context, *bench.Bench) error { return nil } +func (*workload) Iterate(context.Context, *bench.Bench) error { return nil } +func (*workload) Teardown(context.Context, *bench.Bench) error { return nil } func main() { - _ = report.SchemaVersion + stroppy.Main(func() bench.Workload { return &workload{} }) } From b2113954b84d8eaff9e6c238dfee32c72355ec6c Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Mon, 28 Sep 2026 23:41:39 +0300 Subject: [PATCH 09/14] docs(workload): document standalone applications --- CHANGELOG.md | 5 ++++ README.md | 3 +++ docs/run-reports.md | 9 ++++--- docs/standalone-workloads.md | 52 ++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 docs/standalone-workloads.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e0952670..eb5f1b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Group lines under `Added` / `Changed` / `Fixed` / `Removed`. Append a PR link ## [Unreleased] +### Added + +- Custom Go workloads can run as standalone Stroppy applications with the standard drivers, CLI, probe, version, reports, and programmatic runtime API. +- Every constructed workload run report is saved under `~/.stroppy/reports/`; history write failures warn without failing the run. + ### Changed - Go applications can depend on Stroppy v6 through the standard `github.com/stroppy-io/stroppy/v6` module path. ([#181](https://github.com/stroppy-io/stroppy/pull/181)) diff --git a/README.md b/README.md index 5d81331c..d59ddf88 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,9 @@ Human output and diagnostics remain on stderr, so requested stdout stays valid JSON. See [run reports](docs/run-reports.md) for report fields, output behavior, and workload payloads. +Custom Go workloads can also run as self-contained applications with `go run .` +or a normal built executable. See [standalone workloads](docs/standalone-workloads.md). + Use `stroppy help` to explore available topics: ```bash diff --git a/docs/run-reports.md b/docs/run-reports.md index b330d8f3..d660e3dd 100644 --- a/docs/run-reports.md +++ b/docs/run-reports.md @@ -19,9 +19,12 @@ stroppy run simple -d noop --no-report ``` `--report-format` currently accepts `json`. `--report-file` publishes the file -atomically. A requested output failure returns a nonzero exit status. When a run -itself fails after reporting starts, Stroppy still writes the requested report -with `status: "failed"` or `status: "canceled"`, then returns the run error. +atomically. A requested output failure returns a nonzero exit status. Every +constructed report is also saved under `~/.stroppy/reports/`; `--no-report` +disables both report construction and automatic history. History failures warn +on stderr without failing the run. When a run itself fails after reporting starts, +Stroppy still writes the requested report with `status: "failed"` or +`status: "canceled"`, then returns the run error. Common fields include report schema and identity, Stroppy version, timestamps, host runtime facts, driver types, effective scenario and parameter values, diff --git a/docs/standalone-workloads.md b/docs/standalone-workloads.md new file mode 100644 index 00000000..39b08baf --- /dev/null +++ b/docs/standalone-workloads.md @@ -0,0 +1,52 @@ +# Standalone Go workloads + +A standalone workload is one ordinary Go project. It imports Stroppy, defines one +`bench.Workload`, and hands a factory to `stroppy.Main`: + +```go +package main + +import ( + "context" + + stroppy "github.com/stroppy-io/stroppy/v6" + "github.com/stroppy-io/stroppy/v6/pkg/bench" +) + +type workload struct{} + +func (*workload) Name() string { return "example/query" } +func (*workload) Define(*bench.Def) error { return nil } +func (*workload) Setup(context.Context, *bench.Bench) error { return nil } +func (*workload) Iterate(context.Context, *bench.Bench) error { return nil } +func (*workload) Teardown(context.Context, *bench.Bench) error { return nil } + +func main() { + stroppy.Main(func() bench.Workload { return &workload{} }) +} +``` + +Run directly with normal workload flags; workload name is implicit: + +```bash +go run . -d noop --iterations 10 +go run . --help +go run . probe -o json +go run . version --json +``` + +`go build` creates a self-contained executable with Stroppy's repository-defined +PostgreSQL, MySQL, Picodata, YDB, noop, and CSV drivers. No driver blank imports, +installed `stroppy` command, or imports from `cmd/` and `internal/` are required. + +Standalone runs use the same configuration, metrics, report, cancellation, and +`~/.stroppy` storage conventions as installed Stroppy. Constructed run reports +are saved under `~/.stroppy/reports/`; `--no-report` disables both construction +and history. Explicit `--report-file` write failures fail the command. Automatic +history failures only warn on stderr. + +Applications with their own CLI, UI, or service can use `stroppy.New`, +`Application.Execute`, and `Application.Run` instead of `stroppy.Main`. + +This API remains provisional until the workload API stabilization work in #179 +is complete. From 27fdaecd9dc955a5875887e3340183b6ab6d6376 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Tue, 29 Sep 2026 00:08:09 +0300 Subject: [PATCH 10/14] fix(stroppy): harden standalone runtime contracts --- cmd/stroppy/commands/run/run.go | 4 +++ pkg/bench/catalog.go | 4 +++ pkg/bench/root.go | 2 ++ pkg/bench/runtime.go | 2 ++ pkg/report/history.go | 12 +++++++-- pkg/report/history_test.go | 4 +++ stroppy.go | 43 ++++++++++++++++++++++---------- stroppy_test.go | 6 ++++- test/modulecompat/module_test.go | 13 ++++++++-- 9 files changed, 72 insertions(+), 18 deletions(-) diff --git a/cmd/stroppy/commands/run/run.go b/cmd/stroppy/commands/run/run.go index 28bc195b..1c582678 100644 --- a/cmd/stroppy/commands/run/run.go +++ b/cmd/stroppy/commands/run/run.go @@ -71,6 +71,8 @@ var Cmd = NewCommand(bench.RegisteredCatalog(), "") // NewCommand builds a run command over an explicit workload catalog. A non-empty // default workload accepts flags without a workload positional. +// +//nolint:gocognit,cyclop,funlen,maintidx // Command wiring keeps parsing, precedence, and dispatch order visible. func NewCommand(catalog *bench.Catalog, defaultWorkload string) *cobra.Command { cmd := &cobra.Command{ Use: "run [] [sql_file] [-f config.json] [-d driver] [-D key=value] " + @@ -530,6 +532,7 @@ func metricsConfig(cfg *config.RunConfig) *bench.MetricsConfig { return metrics } +//nolint:gocognit // Completion mirrors run parsing and parameter projection. func completeRunArgs(catalog *bench.Catalog) cobra.CompletionFunc { return func( _ *cobra.Command, @@ -738,6 +741,7 @@ func runGoWorkload( ) var outputErr error + if runReport != nil { if _, err := report.Save(runReport); err != nil { fmt.Fprintf(cmd.ErrOrStderr(), "warning: save report history: %v\n", err) diff --git a/pkg/bench/catalog.go b/pkg/bench/catalog.go index fa7994e7..95a551a4 100644 --- a/pkg/bench/catalog.go +++ b/pkg/bench/catalog.go @@ -28,6 +28,7 @@ func NewCatalog(factories ...Factory) (*Catalog, error) { for _, factory := range factories { workload := workloadFromFactory(factory) name := workload.Name() + if _, exists := catalog.factories[name]; exists { return nil, fmt.Errorf("%w %q", errDuplicateWorkload, name) } @@ -101,10 +102,12 @@ func (c *Catalog) names() []string { if c.registered { regMu.RLock() + names := make([]string, 0, len(regWorkloads)) for name := range regWorkloads { names = append(names, name) } + regMu.RUnlock() slices.Sort(names) @@ -115,6 +118,7 @@ func (c *Catalog) names() []string { for name := range c.factories { names = append(names, name) } + slices.Sort(names) return names diff --git a/pkg/bench/root.go b/pkg/bench/root.go index 6d9920bc..5239883d 100644 --- a/pkg/bench/root.go +++ b/pkg/bench/root.go @@ -66,9 +66,11 @@ func newRootState( var quiet bool summaryWriter := io.Writer(os.Stderr) + if metricsConfig != nil { onSummary = metricsConfig.OnSummary quiet = metricsConfig.Quiet + if metricsConfig.SummaryWriter != nil { summaryWriter = metricsConfig.SummaryWriter } diff --git a/pkg/bench/runtime.go b/pkg/bench/runtime.go index a5ce80ae..8f5205d0 100644 --- a/pkg/bench/runtime.go +++ b/pkg/bench/runtime.go @@ -162,7 +162,9 @@ func DescribeFactory(factory Factory) (Description, error) { func registeredFactory(name string) (Factory, bool) { regMu.RLock() + factory, ok := regWorkloads[name] + regMu.RUnlock() return factory, ok diff --git a/pkg/report/history.go b/pkg/report/history.go index ee456ef5..8ab62b27 100644 --- a/pkg/report/history.go +++ b/pkg/report/history.go @@ -16,12 +16,15 @@ const ( maxNameAttempts = 1000 ) -var errHistoryNameExhausted = errors.New("report history filename suffixes exhausted") +var ( + errHistoryNameExhausted = errors.New("report history filename suffixes exhausted") + errNilRunReport = errors.New("nil run report") +) // Save stores one run report under ~/.stroppy/reports and returns its path. func Save(run *Run) (string, error) { if run == nil { - return "", errors.New("save nil run report") + return "", errNilRunReport } home, err := os.UserHomeDir() @@ -47,21 +50,25 @@ func Save(run *Run) (string, error) { func writeHistoryFile(dir, name string, data []byte) (string, error) { for suffix := 1; suffix <= maxNameAttempts; suffix++ { fileName := name + ".json" + if suffix > 1 { fileName = fmt.Sprintf("%s-%d.json", name, suffix) } path := filepath.Join(dir, fileName) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, historyFilePerm) if errors.Is(err, fs.ErrExist) { continue } + if err != nil { return "", fmt.Errorf("create report history file: %w", err) } _, writeErr := file.Write(data) closeErr := file.Close() + if writeErr != nil || closeErr != nil { removeErr := os.Remove(path) @@ -76,6 +83,7 @@ func writeHistoryFile(dir, name string, data []byte) (string, error) { func safeName(name string) string { var builder strings.Builder + for _, value := range strings.ToLower(name) { switch { case value >= 'a' && value <= 'z', value >= '0' && value <= '9', value == '-', value == '_': diff --git a/pkg/report/history_test.go b/pkg/report/history_test.go index 0faeceac..d0dbda45 100644 --- a/pkg/report/history_test.go +++ b/pkg/report/history_test.go @@ -11,10 +11,12 @@ func TestSaveWritesUniqueRunHistory(t *testing.T) { t.Setenv("HOME", t.TempDir()) run := &Run{StartedAt: time.Date(2026, time.September, 28, 12, 34, 56, 0, time.UTC), Workload: "custom/example"} + first, err := Save(run) if err != nil { t.Fatal(err) } + second, err := Save(run) if err != nil { t.Fatal(err) @@ -25,9 +27,11 @@ func TestSaveWritesUniqueRunHistory(t *testing.T) { } wantDir := filepath.Join(os.Getenv("HOME"), ".stroppy", "reports") + if filepath.Dir(first) != wantDir || filepath.Base(first) != "2026-09-28T12-34-56Z-custom-example.json" { t.Fatalf("first history path = %s", first) } + if filepath.Base(second) != "2026-09-28T12-34-56Z-custom-example-2.json" { t.Fatalf("second history path = %s", second) } diff --git a/stroppy.go b/stroppy.go index 442f2368..d3cf99f6 100644 --- a/stroppy.go +++ b/stroppy.go @@ -31,8 +31,11 @@ type Factory = bench.Factory // Workload is implemented by a Go-native benchmark. type Workload = bench.Workload +const defaultPostgresURL = "postgres://postgres:postgres@localhost:5432" //nolint:gosec // local development default + var ( errReservedWorkloadName = errors.New("stroppy: reserved workload name") + errNilApplication = errors.New("stroppy: nil application") reservedWorkloadNames = map[string]struct{}{"help": {}, "probe": {}, "version": {}} ) @@ -79,7 +82,7 @@ func (a *Application) Execute( stdout, stderr io.Writer, ) error { if a == nil { - return errors.New("stroppy: nil application") + return errNilApplication } return cli.Execute(ctx, cli.Options{ @@ -99,18 +102,18 @@ type RunRequest struct { } // Run executes the standalone workload directly and returns its report. -func (a *Application) Run(ctx context.Context, request RunRequest) (*report.Run, error) { +func (a *Application) Run(ctx context.Context, request *RunRequest) (*report.Run, error) { if a == nil { - return nil, errors.New("stroppy: nil application") + return nil, errNilApplication + } + + if request == nil { + request = &RunRequest{} } drivers := request.Drivers if drivers == nil { - drivers = map[int]*config.DriverConfig{0: { - DriverType: config.DriverTypePostgres, - URL: "postgres://postgres:postgres@localhost:5432", - DefaultInsertMethod: "native", - }} + drivers = defaultDrivers() } log := request.Logger @@ -122,6 +125,7 @@ func (a *Application) Run(ctx context.Context, request RunRequest) (*report.Run, if metrics == nil { metrics = &bench.MetricsConfig{} } + if metrics.ServiceVersion == "" { metrics.ServiceVersion = version.Version } @@ -146,17 +150,30 @@ func (a *Application) Run(ctx context.Context, request RunRequest) (*report.Run, // Main runs one workload as a standalone process with Stroppy signal semantics. func Main(factory Factory) { + os.Exit(mainExitCode(factory)) +} + +func mainExitCode(factory Factory) int { application, err := New(factory) if err != nil { fmt.Fprintln(os.Stderr, err) - os.Exit(1) + + return 1 } ctx, stop, exitStatus := shutdown.NotifyContext(context.Background(), nil) - defer stop() err = application.Execute(ctx, os.Args[1:], os.Stdout, os.Stderr) - if code := cli.ExitCodeFor(exitStatus(), err); code != 0 { - os.Exit(code) - } + + stop() + + return cli.ExitCodeFor(exitStatus(), err) +} + +func defaultDrivers() map[int]*config.DriverConfig { + return map[int]*config.DriverConfig{0: { + DriverType: config.DriverTypePostgres, + URL: defaultPostgresURL, + DefaultInsertMethod: "native", + }} } diff --git a/stroppy_test.go b/stroppy_test.go index 1fd8959f..4354c87e 100644 --- a/stroppy_test.go +++ b/stroppy_test.go @@ -35,7 +35,7 @@ func TestApplicationRun(t *testing.T) { t.Fatal(err) } - report, err := app.Run(t.Context(), stroppy.RunRequest{ + report, err := app.Run(t.Context(), &stroppy.RunRequest{ Drivers: map[int]*config.DriverConfig{0: {DriverType: config.DriverTypeNoop}}, Params: bench.ParamInputs{CLI: map[string]string{"iterations": "2"}}, Metrics: &bench.MetricsConfig{Quiet: true}, @@ -68,9 +68,11 @@ func TestApplicationExecute(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { var stdout, stderr bytes.Buffer + if err := app.Execute(t.Context(), test.args, &stdout, &stderr); err != nil { t.Fatalf("Execute() error = %v, stderr = %q", err, stderr.String()) } + if !strings.Contains(stdout.String(), test.want) { t.Fatalf("stdout = %q, want %q", stdout.String(), test.want) } @@ -78,6 +80,7 @@ func TestApplicationExecute(t *testing.T) { } var stdout, stderr bytes.Buffer + err = app.Execute(t.Context(), []string{ "-d", "noop", "--iterations", "2", "--report-format", "json", }, &stdout, &stderr) @@ -89,6 +92,7 @@ func TestApplicationExecute(t *testing.T) { if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatalf("decode report: %v\n%s", err, stdout.String()) } + if report["workload"] != "external/example" { t.Fatalf("workload = %v", report["workload"]) } diff --git a/test/modulecompat/module_test.go b/test/modulecompat/module_test.go index 4da64fc5..cdd3612a 100644 --- a/test/modulecompat/module_test.go +++ b/test/modulecompat/module_test.go @@ -23,6 +23,7 @@ func TestExternalModuleResolvesV6Release(t *testing.T) { if err != nil { t.Fatal(err) } + proxyDir := t.TempDir() writeModuleProxy(t, proxyDir, repoRoot) @@ -43,6 +44,7 @@ func TestExternalModuleResolvesV6Release(t *testing.T) { home := t.TempDir() runEnv := append(os.Environ(), "HOME="+home) + for _, test := range []struct { args []string want string @@ -112,12 +114,13 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { archive := zip.NewWriter(file) prefix := modulePath + "@" + version + "/" - archived := 0 + err = filepath.WalkDir(repoRoot, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } + if entry.IsDir() { name := entry.Name() if path != repoRoot && (strings.HasPrefix(name, ".") || name == "build" || name == "bin") { @@ -131,6 +134,7 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { if err != nil { return err } + if !moduleArchiveFile(relative) { return nil } @@ -139,10 +143,12 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { if err != nil { return err } + writer, err := archive.Create(prefix + filepath.ToSlash(relative)) if err != nil { return err } + _, err = writer.Write(body) archived++ @@ -151,6 +157,7 @@ func writeModuleProxy(t *testing.T, proxyDir, repoRoot string) { if err != nil { t.Fatal(err) } + if archived == 0 { t.Fatal("module archive is empty") } @@ -187,14 +194,16 @@ func makeWritable(t *testing.T, root string) { err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { - return nil + return walkErr } + if entry.IsDir() { return os.Chmod(path, 0o700) } return os.Chmod(path, 0o600) }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { t.Errorf("make module cache writable: %v", err) } From bfbe8235210ca4bebd2461de246f55291a2e3bb3 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Tue, 29 Sep 2026 00:12:34 +0300 Subject: [PATCH 11/14] fix(version): resolve linked Stroppy modules --- cmd/stroppy/commands/baseline/baseline.go | 2 +- cmd/stroppy/commands/run/run.go | 4 ++-- internal/cli/root.go | 20 +--------------- internal/version/version.go | 28 +++++++++++++++++++++++ internal/version/version_test.go | 13 +++++++++++ pkg/driver/csv/manifest.go | 2 +- stroppy.go | 4 ++-- 7 files changed, 48 insertions(+), 25 deletions(-) create mode 100644 internal/version/version_test.go diff --git a/cmd/stroppy/commands/baseline/baseline.go b/cmd/stroppy/commands/baseline/baseline.go index 035f5fc2..6a8e981c 100644 --- a/cmd/stroppy/commands/baseline/baseline.go +++ b/cmd/stroppy/commands/baseline/baseline.go @@ -202,7 +202,7 @@ func run(ctx context.Context, out io.Writer) error { report := Report{ Schema: reportSchema, - Stroppy: version.Version, + Stroppy: version.Resolve(), Time: time.Now().UTC(), Host: hostInfo(), } diff --git a/cmd/stroppy/commands/run/run.go b/cmd/stroppy/commands/run/run.go index 1c582678..47932150 100644 --- a/cmd/stroppy/commands/run/run.go +++ b/cmd/stroppy/commands/run/run.go @@ -508,7 +508,7 @@ func withoutLoggerEnv(values map[string]string) map[string]string { } func metricsConfig(cfg *config.RunConfig) *bench.MetricsConfig { - metrics := &bench.MetricsConfig{ServiceVersion: version.Version} + metrics := &bench.MetricsConfig{ServiceVersion: version.Resolve()} if cfg == nil || cfg.Global == nil { return metrics } @@ -760,7 +760,7 @@ func runGoWorkload( } func reportOptions(cfg *config.RunConfig) bench.ReportOptions { - options := bench.ReportOptions{StroppyVersion: version.Version} + options := bench.ReportOptions{StroppyVersion: version.Resolve()} if cfg != nil && cfg.Global != nil { options.RunID = cfg.Global.RunID options.Metadata = cfg.Global.Metadata diff --git a/internal/cli/root.go b/internal/cli/root.go index bb6b9a41..65d0d680 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -135,12 +135,8 @@ func newVersionCommand() *cobra.Command { } func componentVersions() map[string]string { - versions := map[string]string{appName: version.Version} + versions := map[string]string{appName: version.Resolve()} if info, ok := debug.ReadBuildInfo(); ok { - if versions[appName] == "unknown" { - versions[appName] = moduleVersion(info) - } - for _, dependency := range info.Deps { if dependency.Path == "github.com/jackc/pgx/v5" { versions["pgx"] = dependency.Version @@ -150,17 +146,3 @@ func componentVersions() map[string]string { return versions } - -func moduleVersion(info *debug.BuildInfo) string { - if info.Main.Version != "" && info.Main.Version != "(devel)" { - return info.Main.Version - } - - for _, dependency := range info.Deps { - if dependency.Path == "github.com/stroppy-io/stroppy/v6" { - return dependency.Version - } - } - - return version.Version -} diff --git a/internal/version/version.go b/internal/version/version.go index 0db118bc..1f05dea8 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -1,3 +1,31 @@ package version +import "runtime/debug" + +const modulePath = "github.com/stroppy-io/stroppy/v6" + var Version = "unknown" + +// Resolve returns the injected Stroppy version, or the linked module version +// when Stroppy runs as a dependency of a standalone workload. +func Resolve() string { + if Version != "" && Version != "unknown" { + return Version + } + + info, ok := debug.ReadBuildInfo() + if !ok { + return Version + } + if info.Main.Path == modulePath && info.Main.Version != "" && info.Main.Version != "(devel)" { + return info.Main.Version + } + + for _, dependency := range info.Deps { + if dependency.Path == modulePath { + return dependency.Version + } + } + + return Version +} diff --git a/internal/version/version_test.go b/internal/version/version_test.go new file mode 100644 index 00000000..410e47df --- /dev/null +++ b/internal/version/version_test.go @@ -0,0 +1,13 @@ +package version + +import "testing" + +func TestResolvePrefersInjectedVersion(t *testing.T) { + previous := Version + Version = "v6.2.3" + t.Cleanup(func() { Version = previous }) + + if got := Resolve(); got != "v6.2.3" { + t.Fatalf("Resolve() = %q", got) + } +} diff --git a/pkg/driver/csv/manifest.go b/pkg/driver/csv/manifest.go index e911f2b8..cf211fe3 100644 --- a/pkg/driver/csv/manifest.go +++ b/pkg/driver/csv/manifest.go @@ -90,7 +90,7 @@ func writeManifest( doc := manifest{ Workload: workloadName, Generated: time.Now().UTC().Format(time.RFC3339), - FrameworkVer: version.Version, + FrameworkVer: version.Resolve(), InsertMethod: "NATIVE", Config: manifestConfig{ Dir: cfg.dir, diff --git a/stroppy.go b/stroppy.go index d3cf99f6..a461b68e 100644 --- a/stroppy.go +++ b/stroppy.go @@ -127,12 +127,12 @@ func (a *Application) Run(ctx context.Context, request *RunRequest) (*report.Run } if metrics.ServiceVersion == "" { - metrics.ServiceVersion = version.Version + metrics.ServiceVersion = version.Resolve() } reportOptions := request.ReportOptions if reportOptions.StroppyVersion == "" { - reportOptions.StroppyVersion = version.Version + reportOptions.StroppyVersion = version.Resolve() } return bench.RunFactoryWithReport( From 1f9d66b399322f44980e21bc8d4235ccc06008b2 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Tue, 29 Sep 2026 00:15:22 +0300 Subject: [PATCH 12/14] style(version): align resolver formatting --- internal/cli/root.go | 1 + internal/version/version.go | 1 + internal/version/version_test.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/cli/root.go b/internal/cli/root.go index 65d0d680..dba365e5 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -136,6 +136,7 @@ func newVersionCommand() *cobra.Command { func componentVersions() map[string]string { versions := map[string]string{appName: version.Resolve()} + if info, ok := debug.ReadBuildInfo(); ok { for _, dependency := range info.Deps { if dependency.Path == "github.com/jackc/pgx/v5" { diff --git a/internal/version/version.go b/internal/version/version.go index 1f05dea8..cbee6e16 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -17,6 +17,7 @@ func Resolve() string { if !ok { return Version } + if info.Main.Path == modulePath && info.Main.Version != "" && info.Main.Version != "(devel)" { return info.Main.Version } diff --git a/internal/version/version_test.go b/internal/version/version_test.go index 410e47df..9c1c20d3 100644 --- a/internal/version/version_test.go +++ b/internal/version/version_test.go @@ -5,6 +5,7 @@ import "testing" func TestResolvePrefersInjectedVersion(t *testing.T) { previous := Version Version = "v6.2.3" + t.Cleanup(func() { Version = previous }) if got := Resolve(); got != "v6.2.3" { From 921d80f180d101766a2bd3ae639e73708e96d890 Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Tue, 29 Sep 2026 00:17:42 +0300 Subject: [PATCH 13/14] docs(changelog): link standalone workload change --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb5f1b6a..6547e9c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ Group lines under `Added` / `Changed` / `Fixed` / `Removed`. Append a PR link ### Added -- Custom Go workloads can run as standalone Stroppy applications with the standard drivers, CLI, probe, version, reports, and programmatic runtime API. -- Every constructed workload run report is saved under `~/.stroppy/reports/`; history write failures warn without failing the run. +- Custom Go workloads can run as standalone Stroppy applications with the standard drivers, CLI, probe, version, reports, and programmatic runtime API. ([#183](https://github.com/stroppy-io/stroppy/pull/183)) +- Every constructed workload run report is saved under `~/.stroppy/reports/`; history write failures warn without failing the run. ([#183](https://github.com/stroppy-io/stroppy/pull/183)) ### Changed From 650ea87a3baaf481371c8188c73286714ba5843c Mon Sep 17 00:00:00 2001 From: Aleksandrov Nikita Date: Tue, 29 Sep 2026 01:51:36 +0300 Subject: [PATCH 14/14] fix(runtime): preserve catalog and metrics identity --- pkg/bench/catalog_test.go | 53 +++++++++++++++++++++++++++++++++++++++ pkg/bench/runtime.go | 25 ++++++++++++------ stroppy.go | 6 ++--- stroppy_test.go | 8 +++++- 4 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 pkg/bench/catalog_test.go diff --git a/pkg/bench/catalog_test.go b/pkg/bench/catalog_test.go new file mode 100644 index 00000000..4f7e71b3 --- /dev/null +++ b/pkg/bench/catalog_test.go @@ -0,0 +1,53 @@ +package bench + +import ( + "context" + "strings" + "testing" + + "go.uber.org/zap" + + "github.com/stroppy-io/stroppy/v6/pkg/config" +) + +func TestRunCatalogRejectsFactoryNameChange(t *testing.T) { + calls := 0 + factory := func() Workload { + calls++ + + name := "test/catalog-name" + if calls > 1 { + name = "test/different-name" + } + + return &catalogTestWorkload{name: name} + } + + catalog, err := NewCatalog(factory) + if err != nil { + t.Fatal(err) + } + + err = RunCatalog( + t.Context(), + catalog, + "test/catalog-name", + map[int]*config.DriverConfig{0: {DriverType: config.DriverTypeNoop}}, + ParamInputs{}, + nil, + nil, + zap.NewNop(), + &MetricsConfig{Quiet: true}, + ) + if err == nil || !strings.Contains(err.Error(), "different name") { + t.Fatalf("RunCatalog() error = %v", err) + } +} + +type catalogTestWorkload struct{ name string } + +func (workload *catalogTestWorkload) Name() string { return workload.name } +func (*catalogTestWorkload) Define(*Def) error { return nil } +func (*catalogTestWorkload) Setup(context.Context, *Bench) error { return nil } +func (*catalogTestWorkload) Iterate(context.Context, *Bench) error { return nil } +func (*catalogTestWorkload) Teardown(context.Context, *Bench) error { return nil } diff --git a/pkg/bench/runtime.go b/pkg/bench/runtime.go index 8f5205d0..1ae05237 100644 --- a/pkg/bench/runtime.go +++ b/pkg/bench/runtime.go @@ -211,7 +211,9 @@ func Run( return fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) } - return RunFactory(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig) + _, err := run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil, name) + + return err } // RunFactory executes one fresh workload from factory. @@ -224,7 +226,7 @@ func RunFactory( lg *zap.Logger, metricsConfig *MetricsConfig, ) error { - _, err := run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil) + _, err := run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil, "") return err } @@ -246,8 +248,8 @@ func RunWithReport( return nil, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) } - return RunFactoryWithReport( - ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, reportOptions, + return run( + ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions, name, ) } @@ -267,7 +269,9 @@ func RunCatalog( return fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) } - return RunFactory(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig) + _, err := run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, nil, name) + + return err } // RunCatalogWithReport executes one fresh workload selected from catalog and returns its report. @@ -287,8 +291,8 @@ func RunCatalogWithReport( return nil, fmt.Errorf("%w as %q", errNoWorkloadRegistered, name) } - return RunFactoryWithReport( - ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, reportOptions, + return run( + ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions, name, ) } @@ -303,7 +307,7 @@ func RunFactoryWithReport( metricsConfig *MetricsConfig, reportOptions ReportOptions, ) (*report.Run, error) { - return run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions) + return run(ctx, factory, drivers, paramInputs, steps, noSteps, lg, metricsConfig, &reportOptions, "") } //nolint:funlen,gocognit // lifecycle order stays explicit: setup, scenario, teardown, report. @@ -316,10 +320,15 @@ func run( lg *zap.Logger, metricsConfig *MetricsConfig, reportOptions *ReportOptions, + expectedName string, ) (*report.Run, error) { wl := workloadFromFactory(factory) name := wl.Name() + if expectedName != "" && name != expectedName { + return nil, fmt.Errorf("%w: got %q, want %q", errFactoryNameChanged, name, expectedName) + } + scenarioParams, definition, err := defineWorkload(wl, paramInputs, false) if err != nil { return nil, fmt.Errorf("define workload %q: %w", name, err) diff --git a/stroppy.go b/stroppy.go index a461b68e..4310a9a5 100644 --- a/stroppy.go +++ b/stroppy.go @@ -121,9 +121,9 @@ func (a *Application) Run(ctx context.Context, request *RunRequest) (*report.Run log = logger.Global() } - metrics := request.Metrics - if metrics == nil { - metrics = &bench.MetricsConfig{} + metrics := &bench.MetricsConfig{} + if request.Metrics != nil { + *metrics = *request.Metrics } if metrics.ServiceVersion == "" { diff --git a/stroppy_test.go b/stroppy_test.go index 4354c87e..d6b5a310 100644 --- a/stroppy_test.go +++ b/stroppy_test.go @@ -35,10 +35,12 @@ func TestApplicationRun(t *testing.T) { t.Fatal(err) } + metrics := &bench.MetricsConfig{Quiet: true} + report, err := app.Run(t.Context(), &stroppy.RunRequest{ Drivers: map[int]*config.DriverConfig{0: {DriverType: config.DriverTypeNoop}}, Params: bench.ParamInputs{CLI: map[string]string{"iterations": "2"}}, - Metrics: &bench.MetricsConfig{Quiet: true}, + Metrics: metrics, }) if err != nil { t.Fatal(err) @@ -47,6 +49,10 @@ func TestApplicationRun(t *testing.T) { if report.Workload != "external/example" || report.Driver != "noop" { t.Fatalf("report identity = %s/%s", report.Workload, report.Driver) } + + if metrics.ServiceVersion != "" { + t.Fatalf("caller metrics service version mutated to %q", metrics.ServiceVersion) + } } func TestApplicationExecute(t *testing.T) {