diff --git a/GNUmakefile b/GNUmakefile index a5667cc5c86..3d7fb03dc1c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -224,7 +224,7 @@ gomodslocalupdate: gomods ## Run gomod-local-update .PHONY: mockery mockery: $(mockery) ## Install mockery. - go install github.com/vektra/mockery/v2@v2.53.0 + GOTOOLCHAIN=go$(shell awk '/^go /{print $$2}' go.mod) go install github.com/vektra/mockery/v2@v2.53.0 .PHONY: codecgen codecgen: $(codecgen) ## Install codecgen diff --git a/core/cmd/shell_local_test.go b/core/cmd/shell_local_test.go index 5756715354a..1438c275a0b 100644 --- a/core/cmd/shell_local_test.go +++ b/core/cmd/shell_local_test.go @@ -75,7 +75,7 @@ func genTestEVMRelayers(t *testing.T, cfg chainlink.GeneralConfig, ds sqlutil.Da f := chainlink.RelayerFactory{ Logger: lggr, LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), - cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), nil, "", cfg.LOOPP()), + cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), cfg.Metering(), nil, "", cfg.LOOPP()), CapabilitiesRegistry: capabilities.NewRegistry(lggr), } diff --git a/core/config/app_config.go b/core/config/app_config.go index 7965d096c73..9ace57f66d1 100644 --- a/core/config/app_config.go +++ b/core/config/app_config.go @@ -61,6 +61,7 @@ type AppConfig interface { WebServer() WebServer Tracing() Tracing Telemetry() Telemetry + Metering() Metering CRE() CRE CCV() CCV Billing() Billing diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index ac8f8c472ee..9c54131d6a7 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -951,6 +951,28 @@ Enabled = false # Default # By default, we only forward the go runtime metrics. Empty means forward everything. Prefixes = ["go_"] # Default +# Metering configures durable resource metering emission and the coarse +# deployment/node identity dimensions stamped on emitted MeterRecords and +# MeterSnapshots. +[Metering] +# MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. +MeterRecordsEnabled = false # Default +# MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +# Requires MeterRecordsEnabled = true. +MeterSnapshotsEnabled = false # Default +# Product is the deployment product identity dimension, e.g. 'cre'. +Product = 'cre' # Default +# Tenant is the human-readable tenant name, e.g. 'mainline'. +Tenant = '' # Default +# NumericTenantID is the numbered tenant identifier represented as a string. +NumericTenantID = '' # Default +# Environment is the deployment environment identity dimension, e.g. 'production'. +Environment = '' # Default +# Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. +Zone = '' # Default +# NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). +NodeID = '' # Default + [CRE.Streams] # WsURL is the websockets url for the streams sdk config WsURL = "streams.url" # Example diff --git a/core/config/metering_config.go b/core/config/metering_config.go new file mode 100644 index 00000000000..9f11e1c3419 --- /dev/null +++ b/core/config/metering_config.go @@ -0,0 +1,16 @@ +package config + +// Metering exposes durable resource-metering configuration: the emission +// toggles and the coarse deployment/node identity dimensions stamped on emitted +// MeterRecords and MeterSnapshots. These are passed via loop.EnvConfig to every LOOP +// plugin. +type Metering interface { + MeterRecordsEnabled() bool + MeterSnapshotsEnabled() bool + Product() string + Tenant() string + NumericTenantID() string + Environment() string + Zone() string + NodeID() string +} diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 7f9e3d4bb5a..1c449f3fea2 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -65,6 +65,7 @@ type Core struct { Mercury Mercury `toml:",omitempty"` Capabilities Capabilities `toml:",omitempty"` Telemetry Telemetry `toml:",omitempty"` + Metering Metering `toml:",omitempty"` Workflows Workflows `toml:",omitempty"` CRE CreConfig `toml:",omitempty"` Billing Billing `toml:",omitempty"` @@ -113,6 +114,7 @@ func (c *Core) SetFrom(f *Core) { c.JobDistributor.setFrom(&f.JobDistributor) c.Tracing.setFrom(&f.Tracing) c.Telemetry.setFrom(&f.Telemetry) + c.Metering.setFrom(&f.Metering) c.CRE.setFrom(&f.CRE) c.Billing.setFrom(&f.Billing) c.BridgeStatusReporter.setFrom(&f.BridgeStatusReporter) @@ -3164,6 +3166,75 @@ func (b *Telemetry) ValidateConfig() (err error) { return err } +// Metering configures durable resource metering emission and the coarse +// deployment/node identity dimensions stamped on emitted MeterRecords and +// MeterSnapshots. These are passed via loop.EnvConfig to every LOOP plugin. +type Metering struct { + // MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + MeterRecordsEnabled *bool + // MeterSnapshotsEnabled enables durable MeterSnapshot emission. Requires + // MeterRecordsEnabled to be true. + MeterSnapshotsEnabled *bool + // Product is the deployment product identity dimension, e.g. "cre". + Product *string + // Tenant is the human-readable tenant name, e.g. "mainline". + Tenant *string + // NumericTenantID is the numbered tenant identifier as a string. + NumericTenantID *string + // Environment is the deployment environment dimension, e.g. "production". + Environment *string + // Zone is the deployment zone dimension, e.g. "wf-zone-a". + Zone *string + // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1" (NOT the CSA + // public key) + NodeID *string +} + +func (b *Metering) setFrom(f *Metering) { + if v := f.MeterRecordsEnabled; v != nil { + b.MeterRecordsEnabled = v + } + if v := f.MeterSnapshotsEnabled; v != nil { + b.MeterSnapshotsEnabled = v + } + if v := f.Product; v != nil { + b.Product = v + } + if v := f.Tenant; v != nil { + b.Tenant = v + } + if v := f.NumericTenantID; v != nil { + b.NumericTenantID = v + } + if v := f.Environment; v != nil { + b.Environment = v + } + if v := f.Zone; v != nil { + b.Zone = v + } + if v := f.NodeID; v != nil { + b.NodeID = v + } +} + +func (b *Metering) ValidateConfig() (err error) { + if b.MeterSnapshotsEnabled != nil && *b.MeterSnapshotsEnabled && (b.MeterRecordsEnabled == nil || !*b.MeterRecordsEnabled) { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "MeterSnapshotsEnabled", + Value: true, + Msg: "requires MeterRecordsEnabled to be true", + }) + } + if b.MeterRecordsEnabled != nil && *b.MeterRecordsEnabled && (b.NodeID == nil || *b.NodeID == "") { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "NodeID", + Value: "", + Msg: "must be non-empty when MeterRecordsEnabled is true (an empty NodeID collapses per-node snapshot dedup scope DON-wide)", + }) + } + return err +} + type PrometheusBridge struct { Enabled *bool Prefixes []string diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index bae3d4b1be6..2766aa1570e 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -860,3 +860,65 @@ func durationPtr(d time.Duration) *commonconfig.Duration { cd := *commonconfig.MustNewDuration(d) return &cd } + +func TestMetering_ValidateConfig(t *testing.T) { + testCases := []struct { + name string + config *Metering + expectError bool + errorMsg string + }{ + { + name: "disabled with all nil fields", + config: &Metering{}, + expectError: false, + }, + { + name: "records enabled with non-empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: false, + }, + { + name: "records enabled with nil NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: nil, + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "records enabled with empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new(""), + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "snapshots enabled without records enabled", + config: &Metering{ + MeterSnapshotsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: true, + errorMsg: "requires MeterRecordsEnabled to be true", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.config.ValidateConfig() + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml index 0401c754ad2..2c39f14fd45 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml @@ -40,7 +40,7 @@ name = "workflow" don_family = "test-don-family" don_types = ["workflow"] - override_mode = "all" + override_mode = "each" http_port_range_start = 10100 # even though this DON is not using any capability for chain with ID 2337 we still need it to be connected to it, @@ -57,14 +57,104 @@ image = "postgres:12.0" port = 13000 - [[nodesets.node_specs]] + # override_mode = "each": one node_specs block per node so each node gets a + # distinct [Metering].NodeID (node_id must be unique per node for snapshot + # dedup / cross-node median). Keep count in sync with nodes = 4. + [[nodesets.node_specs]] roles = ["plugin"] [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "core/chainlink.Dockerfile" docker_build_args = { "CL_IS_PROD_BUILD" = "false" } # image = "chainlink-tmp:latest" - user_config_overrides = "" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-1' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-2' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-3' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-4' + """ [[nodesets]] nodes = 4 @@ -72,7 +162,7 @@ don_family = "test-don-family" don_types = ["capabilities"] exposes_remote_capabilities = true - override_mode = "all" + override_mode = "each" http_port_range_start = 10200 # we need to have chain 1337 configured (even if no capability uses it), because we use node addresses on chain 1337 @@ -86,6 +176,8 @@ image = "postgres:12.0" port = 13100 + # override_mode = "each": one node_specs block per node so each node gets a + # distinct [Metering].NodeID. Keep count in sync with nodes = 4. [[nodesets.node_specs]] roles = ["plugin"] [nodesets.node_specs.node] @@ -93,7 +185,94 @@ docker_file = "core/chainlink.Dockerfile" docker_build_args = { "CL_IS_PROD_BUILD" = "false" } # image = "chainlink-tmp:latest" - user_config_overrides = "" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-1' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-2' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-3' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-4' + """ [[nodesets]] nodes = 1 diff --git a/core/scripts/cre/environment/environment/chip_ingress_stack.go b/core/scripts/cre/environment/environment/chip_ingress_stack.go index eaac04360ef..f3d3bfca0fa 100644 --- a/core/scripts/cre/environment/environment/chip_ingress_stack.go +++ b/core/scripts/cre/environment/environment/chip_ingress_stack.go @@ -77,10 +77,14 @@ func schemaCommitRefFromGoMod(ctx context.Context, repoRoot, targetModule string // getSchemaSetFromGoMod resolves SchemaSets from chainlink-protos commits pinned in go.mod: // - workflows (chip-cre.json) for CRE/workflow telemetry // - node-platform (chip-schemas.json) for PluginRelayerConfigEmitter / common.v1.ChainPluginConfig +// - metering (chip-cll.meter.json) for durable resource metering (MeterRecord/MeterSnapshot on +// the cll.meter domain); without this, ChIP Ingress rejects those events at pre-publish encode +// time with "Subject 'cll-meter-metering.v1.MeterRecord' not found" and drops them silently. func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, error) { const ( workflowsModule = "github.com/smartcontractkit/chainlink-protos/workflows/go" nodePlatformModule = "github.com/smartcontractkit/chainlink-protos/node-platform" + meteringModule = "github.com/smartcontractkit/chainlink-protos/metering/go" ) repoRoot, err := filepath.Abs(relativePathToRepoRoot) @@ -100,6 +104,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err } framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", nodePlatformModule, npRef, npVer) + meteringRef, meteringVer, err := schemaCommitRefFromGoMod(ctx, repoRoot, meteringModule) + if err != nil { + return nil, err + } + framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", meteringModule, meteringRef, meteringVer) + return []chipingressset.SchemaSet{ { URI: chainlinkProtosGitURI, @@ -113,6 +123,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err SchemaDir: "node-platform", ConfigFile: "chip-schemas.json", }, + { + URI: chainlinkProtosGitURI, + Ref: meteringRef, + SchemaDir: "metering", + ConfigFile: "chip-cll.meter.json", + }, }, nil } @@ -1045,6 +1061,6 @@ func fetchAndRegisterProtosCmd() *cobra.Command { }) }, } - cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "h", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") + cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "u", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") return cmd } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 1c021b4b12e..9faf1f66fbb 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -511,7 +511,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 6065a6db965..344e16ea697 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1626,8 +1626,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 59e41aa6870..fad174da69a 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -289,7 +289,7 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err } loopRegistry := plugins.NewLoopRegistry(globalLogger, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), - beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) + cfg.Metering(), beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) relayerFactory := RelayerFactory{ Logger: opts.Logger, diff --git a/core/services/chainlink/config_general.go b/core/services/chainlink/config_general.go index 77e72388d1b..7d3cc03ad39 100644 --- a/core/services/chainlink/config_general.go +++ b/core/services/chainlink/config_general.go @@ -588,6 +588,10 @@ func (g *generalConfig) Telemetry() coreconfig.Telemetry { return &telemetryConfig{s: g.c.Telemetry} } +func (g *generalConfig) Metering() coreconfig.Metering { + return &meteringConfig{s: g.c.Metering} +} + func (g *generalConfig) CRE() coreconfig.CRE { return &creConfig{s: g.secrets.CRE, c: g.c.CRE} } diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go new file mode 100644 index 00000000000..a36dbb9e597 --- /dev/null +++ b/core/services/chainlink/config_metering.go @@ -0,0 +1,69 @@ +package chainlink + +import ( + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +type meteringConfig struct { + s toml.Metering +} + +func (b *meteringConfig) MeterRecordsEnabled() bool { + if b.s.MeterRecordsEnabled == nil { + return false + } + return *b.s.MeterRecordsEnabled +} + +func (b *meteringConfig) MeterSnapshotsEnabled() bool { + if b.s.MeterSnapshotsEnabled == nil { + return false + } + return *b.s.MeterSnapshotsEnabled +} + +// Product returns the deployment product identity dimension. The parsed config +// defaults it to "cre" via docs.CoreDefaults so metering is never enabled with +// an empty product dimension; a zero-value toml.Metering that has not been run +// through setDefaults returns "unset" (the nil-pointer fallback below). +func (b *meteringConfig) Product() string { + if b.s.Product == nil { + return "unset" + } + return *b.s.Product +} + +func (b *meteringConfig) Tenant() string { + if b.s.Tenant == nil { + return "" + } + return *b.s.Tenant +} + +func (b *meteringConfig) NumericTenantID() string { + if b.s.NumericTenantID == nil { + return "" + } + return *b.s.NumericTenantID +} + +func (b *meteringConfig) Environment() string { + if b.s.Environment == nil { + return "" + } + return *b.s.Environment +} + +func (b *meteringConfig) Zone() string { + if b.s.Zone == nil { + return "" + } + return *b.s.Zone +} + +func (b *meteringConfig) NodeID() string { + if b.s.NodeID == nil { + return "" + } + return *b.s.NodeID +} diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go new file mode 100644 index 00000000000..71eb6eb0da7 --- /dev/null +++ b/core/services/chainlink/config_metering_test.go @@ -0,0 +1,52 @@ +package chainlink + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +func TestMeteringConfig(t *testing.T) { + t.Parallel() + t.Run("defaults", func(t *testing.T) { + t.Parallel() + mc := meteringConfig{s: toml.Metering{}} + assert.False(t, mc.MeterRecordsEnabled()) + assert.False(t, mc.MeterSnapshotsEnabled()) + // A zero-value toml.Metering (not run through setDefaults) has a nil + // Product pointer, so Product() returns "unset". The parsed config + // applies a "cre" default via docs.CoreDefaults (covered by the + // LogConfiguration effective-TOML test), so metering is never enabled + // with an empty product dimension. + assert.Equal(t, "unset", mc.Product()) + assert.Empty(t, mc.Tenant()) + assert.Empty(t, mc.NumericTenantID()) + assert.Empty(t, mc.Environment()) + assert.Empty(t, mc.Zone()) + assert.Empty(t, mc.NodeID()) + }) + + t.Run("explicit values", func(t *testing.T) { + t.Parallel() + mc := meteringConfig{s: toml.Metering{ + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), + }} + assert.True(t, mc.MeterRecordsEnabled()) + assert.True(t, mc.MeterSnapshotsEnabled()) + assert.Equal(t, "cre", mc.Product()) + assert.Equal(t, "mainline", mc.Tenant()) + assert.Equal(t, "42", mc.NumericTenantID()) + assert.Equal(t, "production", mc.Environment()) + assert.Equal(t, "wf-zone-a", mc.Zone()) + assert.Equal(t, "clp-cre-wf-zone-a-1", mc.NodeID()) + }) +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 1ca79e550ba..43f73ac84ea 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -591,6 +591,16 @@ func TestConfig_Marshal(t *testing.T) { Prefixes: []string{"ocr_"}, }, } + full.Metering = toml.Metering{ + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), + } full.CRE = toml.CreConfig{ UseLocalTimeProvider: new(true), EnableDKGRecipient: new(false), diff --git a/core/services/chainlink/mocks/general_config.go b/core/services/chainlink/mocks/general_config.go index fb6219ca3fc..20e095505b7 100644 --- a/core/services/chainlink/mocks/general_config.go +++ b/core/services/chainlink/mocks/general_config.go @@ -1518,6 +1518,53 @@ func (_c *GeneralConfig_Mercury_Call) RunAndReturn(run func() de.Mercury) *Gener return _c } +// Metering provides a mock function with no fields +func (_m *GeneralConfig) Metering() config.Metering { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Metering") + } + + var r0 config.Metering + if rf, ok := ret.Get(0).(func() config.Metering); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(config.Metering) + } + } + + return r0 +} + +// GeneralConfig_Metering_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Metering' +type GeneralConfig_Metering_Call struct { + *mock.Call +} + +// Metering is a helper method to define mock.On call +func (_e *GeneralConfig_Expecter) Metering() *GeneralConfig_Metering_Call { + return &GeneralConfig_Metering_Call{Call: _e.mock.On("Metering")} +} + +func (_c *GeneralConfig_Metering_Call) Run(run func()) *GeneralConfig_Metering_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *GeneralConfig_Metering_Call) Return(_a0 config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GeneralConfig_Metering_Call) RunAndReturn(run func() config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(run) + return _c +} + // OCR provides a mock function with no fields func (_m *GeneralConfig) OCR() config.OCR { ret := _m.Called() diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index f0e139720dd..cc77a2a4f6d 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index e72250d2270..f12e60a6c0b 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -416,6 +416,16 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +NumericTenantID = '42' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'clp-cre-wf-zone-a-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 049e0576280..6b233347d23 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/cre/confidential_relay_peerid_test.go b/core/services/cre/confidential_relay_peerid_test.go index 8b92cfeaeec..f81afc0393b 100644 --- a/core/services/cre/confidential_relay_peerid_test.go +++ b/core/services/cre/confidential_relay_peerid_test.go @@ -51,6 +51,7 @@ func (s stubConfig) Workflows() config.Workflows { return nil } func (s stubConfig) CRE() config.CRE { return nil } func (s stubConfig) P2P() config.P2P { return s.p2p } func (s stubConfig) Sharding() config.Sharding { return nil } +func (s stubConfig) Metering() config.Metering { return nil } func peerIDFromByte(b byte) p2pkey.PeerID { var id p2pkey.PeerID diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 6cd2ef123e5..b2c3b9a299f 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -18,11 +18,13 @@ import ( "github.com/smartcontractkit/chainlink-common/keystore/corekeys/p2pkey" "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/billing" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" nodeauthjwt "github.com/smartcontractkit/chainlink-common/pkg/nodeauth/jwt" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -297,6 +299,10 @@ func (s *Services) newSubservices( return srvs, nil } + // Build the syncer's base metering identity once from node metering config; + // the handler resolves the workflow DON id later from the don notifier. + meterIdentity := newSyncerMeterIdentity(cfg) + wfSyncer, billingClient, wfSyncerSrvcs, err := newWorkflowRegistrySyncer( cfg, relayerChainInterops, @@ -310,6 +316,7 @@ func (s *Services) newSubservices( opts.LimitsFactory, s.OrgResolver, s.GatewayConnectorWrapper, + meterIdentity, ) if err != nil { return nil, err @@ -338,6 +345,7 @@ type Config interface { CRE() config.CRE P2P() config.P2P Sharding() config.Sharding + Metering() config.Metering } // RelayerChainInterops is the minimal interface needed for relayer chain interops @@ -721,6 +729,30 @@ func newBillingClient(lggr logger.Logger, cfg Config, opts Opts) (metering.Billi return billing.NewWorkflowClient(lggr, cfg.Billing().URL(), workflowOpts...) } +// newSyncerMeterIdentity builds the syncer's base metering identity from node +// metering config. Product/tenant/environment/zone and logical node_id are read +// from [Metering]. The workflow DON id (don_id) is resolved later by the +// handler from the don notifier (the engine runs on the workflow DON), so it is +// intentionally left empty here. Service/resource_pool are stamped by +// syncer.WithIdentity. +func newSyncerMeterIdentity(cfg Config) resourcemanager.ResourceIdentity { + m := cfg.Metering() + if m == nil { + return resourcemanager.ResourceIdentity{} + } + id := resourcemanager.ResourceIdentity{ + Product: m.Product(), + Tenant: m.Tenant(), + NumericTenantID: m.NumericTenantID(), + Environment: m.Environment(), + Zone: m.Zone(), + } + if nodeID := m.NodeID(); nodeID != "" { + id.Don = &resourcemanager.DonIdentity{NodeID: nodeID} + } + return id +} + func newShardOrchestratorClient(cfg Config, lggr logger.Logger) (*shardorchestrator.Client, error) { shardID := cfg.Sharding().ShardIndex() if shardID == 0 { @@ -936,6 +968,7 @@ func newWorkflowRegistrySyncerV2( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, []commonsrv.Service, error) { capCfg := cfg.Capabilities() wfReg := capCfg.WorkflowRegistry() @@ -1012,6 +1045,10 @@ func newWorkflowRegistrySyncerV2( shardRoutingSteady = shardownership.NewSteadySignal(shardownership.WithSteadySignalMetrics(steadyMetrics)) } + meteringCfg := cfg.Metering() + meterRecordsEnabled := meteringCfg != nil && meteringCfg.MeterRecordsEnabled() + meterSnapshotsEnabled := meteringCfg != nil && meteringCfg.MeterSnapshotsEnabled() + handlerOpts := []syncerV2.EventHandlerOption{ syncerV2.WithBillingClient(billingClient), syncerV2.WithWorkflowRegistry(capCfg.WorkflowRegistry().Address(), selector), @@ -1020,6 +1057,13 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithLocalSecretOverrides(lggr, cfg.CRE().LocalSecretOverrides()), syncerV2.WithShardExecutionGuard(shardOrchestratorClient, shardingEnabled, shardIndex), syncerV2.WithShardRoutingSteady(shardRoutingSteady), + syncerV2.WithResourceManager(resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: meterRecordsEnabled, + MeterSnapshotsEnabled: meterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + })), + syncerV2.WithIdentity(meterIdentity), } mc := capCfg.WorkflowRegistry().ModuleCache() @@ -1171,6 +1215,7 @@ func newWorkflowRegistrySyncer( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, metering.BillingClient, []commonsrv.Service, error) { capCfg := cfg.Capabilities() @@ -1219,6 +1264,7 @@ func newWorkflowRegistrySyncer( lf, orgResolver, gatewayConnectorWrapper, + meterIdentity, ) return syncer, billingClient, srvcs, err default: diff --git a/core/services/job/models.go b/core/services/job/models.go index a669cd947ea..3dd92549e30 100644 --- a/core/services/job/models.go +++ b/core/services/job/models.go @@ -843,6 +843,7 @@ type WorkflowSpec struct { UpdatedAt time.Time `toml:"-" db:"updated_at"` SpecType WorkflowSpecType `toml:"spec_type" db:"spec_type"` Attributes []byte `db:"attributes"` + RegisteredAt int64 `toml:"-" db:"registered_at"` sdkWorkflow *sdk.WorkflowSpec rawSpec []byte config []byte diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index 1eb9d655d38..fc6b7d7a324 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -102,6 +102,24 @@ func NewStandardCapabilities( } } +// initialiseDependencies builds the StandardCapabilitiesDependencies delivered to +// the capability LOOP via Initialise. +func (s *StandardCapabilities) initialiseDependencies() core.StandardCapabilitiesDependencies { + return core.StandardCapabilitiesDependencies{ + Config: s.config, + Store: s.store, + CapabilityRegistry: s.CapabilitiesRegistry, + RelayerSet: s.relayerSet, + OracleFactory: s.oracleFactory, + GatewayConnector: s.gatewayConnector, + P2PKeystore: s.keystore, + OrgResolver: s.orgResolver, + CRESettings: s.creSettings, + TriggerEventStore: s.triggerEventStore, + CapabilityDonID: s.capabilityDonID, + } +} + func (s *StandardCapabilities) Start(ctx context.Context) error { return s.StartOnce("StandardCapabilities", func() error { envVars, err := plugins.ParseEnvFile(env.CapabilitiesPlugin.Env.Get()) @@ -138,19 +156,7 @@ func (s *StandardCapabilities) Start(ctx context.Context) error { return } - dependencies := core.StandardCapabilitiesDependencies{ - Config: s.config, - Store: s.store, - CapabilityRegistry: s.CapabilitiesRegistry, - RelayerSet: s.relayerSet, - OracleFactory: s.oracleFactory, - GatewayConnector: s.gatewayConnector, - P2PKeystore: s.keystore, - OrgResolver: s.orgResolver, - CRESettings: s.creSettings, - TriggerEventStore: s.triggerEventStore, - CapabilityDonID: s.capabilityDonID, - } + dependencies := s.initialiseDependencies() if err = s.capabilitiesLoop.Service.Initialise(cctx, dependencies); err != nil { s.log.Errorf("error initialising standard capabilities service: %v", err) s.setReadyErr(fmt.Errorf("initialising standard capabilities service: %w", err)) diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index ce0159b54f7..d9c24e98c9b 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -106,6 +106,66 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { }) } +func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { + t.Parallel() + want := core.StandardCapabilitiesDependencies{ + Config: "test-config", + } + + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + want.Config, + &capturingRegistrar{}, + want, + ) + + got := std.initialiseDependencies() + + require.Equal(t, want.Config, got.Config, "config should be re-delivered to LOOP Initialise dependencies") +} + +// TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP asserts that the +// host-resolved capability DON ID is carried on the dependencies delivered to +// the capability LOOP at Initialise. Without this, a trigger producer silently +// falls back to the consumer workflow's DON for metering identity and event +// labels even when the node knows which DON it is serving. +func TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP(t *testing.T) { + t.Parallel() + t.Run("nonzero DON ID round-trips when the DON is known", func(t *testing.T) { + t.Parallel() + const knownDonID = uint32(42) + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{CapabilityDonID: knownDonID}, + ) + + got := std.initialiseDependencies() + + require.Equal(t, knownDonID, got.CapabilityDonID, + "the host-resolved capability DON ID must reach the LOOP at Initialise") + }) + + t.Run("zero DON ID is preserved when the host could not resolve one", func(t *testing.T) { + t.Parallel() + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{}, + ) + + got := std.initialiseDependencies() + + require.Zero(t, got.CapabilityDonID, + "an unresolved DON ID stays zero so the LOOP can fall back to the workflow DON") + }) +} + const capturingRegistrarErr = "capturingRegistrar: stop after capture" // capturingRegistrar records the CmdConfig handed to RegisterLOOP and then diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 31395bce8f3..8eb7a05b91b 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -3,6 +3,7 @@ package v2 import ( "context" "database/sql" + "errors" "time" "github.com/jmoiron/sqlx" @@ -20,8 +21,24 @@ type WorkflowSpecsDS interface { // GetWorkflowSpecByID returns the workflow spec for the given workflowID. GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) - // DeleteWorkflowSpec deletes the workflow spec for the given workflow ID. - DeleteWorkflowSpec(ctx context.Context, id string) error + // GetWorkflowSpecList returns the persisted workflow specs. It projects only + // the identity columns needed by the metering snapshot path (workflow_id and + // workflow_owner); other fields are left zero. + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) + + // DeleteWorkflowSpec deletes the spec row for id. Idempotent: a missing row + // returns (nil, nil), not an error. The deleted row is returned so the + // caller can derive the generation-scoped metering event_id from the row's + // own persisted registered_at (every node emits the identical id regardless + // of local staleness). + DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) + + // PauseWorkflowSpec tombstones the spec row: status becomes paused and the + // heavy artifact payload (hex binary + config) is cleared in the same + // statement, freeing storage while the row itself remains the durable record + // that this registration generation is still held (level-neutral for + // metering). Idempotent; pausing an absent or already-paused row is a no-op. + PauseWorkflowSpec(ctx context.Context, id string) error // DeleteWorkflowSpecs deletes workflow specs for the given workflow IDs in a single query. DeleteWorkflowSpecs(ctx context.Context, ids []string) error @@ -65,7 +82,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at, updated_at, spec_type, - attributes + attributes, + registered_at ) VALUES ( :workflow, :config, @@ -79,7 +97,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) :created_at, :updated_at, :spec_type, - :attributes + :attributes, + :registered_at ) ON CONFLICT (workflow_id) DO UPDATE SET workflow = EXCLUDED.workflow, @@ -93,7 +112,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, spec_type = EXCLUDED.spec_type, - attributes = EXCLUDED.attributes + attributes = EXCLUDED.attributes, + registered_at = EXCLUDED.registered_at RETURNING id ` @@ -121,6 +141,7 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp ` var spec job.WorkflowSpec + // Note: "Get will return sql.ErrNoRows like row.Scan would" - sqlx@v1.4.0 err := orm.ds.GetContext(ctx, &spec, query, id) if err != nil { return nil, err @@ -129,27 +150,41 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp return &spec, nil } -func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) error { +// GetWorkflowSpecList returns all persisted workflow specs, projecting the +// identity columns needed by the orphan sweep and the metering snapshot path. +func (orm *orm) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { query := ` - DELETE FROM workflow_specs_v2 - WHERE workflow_id = $1 + SELECT workflow_id, workflow_owner, registered_at + FROM workflow_specs_v2 ` - result, err := orm.ds.ExecContext(ctx, query, id) - if err != nil { - return err + var specs []*job.WorkflowSpec + if err := orm.ds.SelectContext(ctx, &specs, query); err != nil { + return nil, err } - rowsAffected, err := result.RowsAffected() - if err != nil { - return err - } + return specs, nil +} - if rowsAffected == 0 { - return sql.ErrNoRows // No spec deleted +func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { + query := `DELETE FROM workflow_specs_v2 WHERE workflow_id = $1 RETURNING *` + var spec job.WorkflowSpec + err := orm.ds.GetContext(ctx, &spec, query, id) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err } + return &spec, nil +} - return nil +func (orm *orm) PauseWorkflowSpec(ctx context.Context, id string) error { + query := `UPDATE workflow_specs_v2 + SET status = $2, workflow = '', config = '', updated_at = now() + WHERE workflow_id = $1` + _, err := orm.ds.ExecContext(ctx, query, id, job.WorkflowSpecStatusPaused) + return err } func (orm *orm) DeleteWorkflowSpecs(ctx context.Context, ids []string) error { diff --git a/core/services/workflows/artifacts/v2/orm_test.go b/core/services/workflows/artifacts/v2/orm_test.go index 63f78c3a1d1..677cdf1cfba 100644 --- a/core/services/workflows/artifacts/v2/orm_test.go +++ b/core/services/workflows/artifacts/v2/orm_test.go @@ -9,6 +9,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -194,8 +195,9 @@ func Test_DeleteWorkflowSpec(t *testing.T) { require.NoError(t, err) require.NotZero(t, id) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) + require.NotNil(t, deletedSpec) // Verify the record is deleted from the database var dbSpec job.WorkflowSpec @@ -204,16 +206,16 @@ func Test_DeleteWorkflowSpec(t *testing.T) { require.Equal(t, sql.ErrNoRows, err) }) - t.Run("fails if no workflow spec exists", func(t *testing.T) { + t.Run("returns false for non-existent workflow spec", func(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) ctx := t.Context() lggr := logger.TestLogger(t) orm := &orm{ds: db, lggr: lggr} - err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") - require.Error(t, err) - require.Equal(t, sql.ErrNoRows, err) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") + require.NoError(t, err) + assert.Nil(t, deletedSpec) }) } @@ -293,7 +295,7 @@ func Test_GetWorkflowSpec(t *testing.T) { require.NoError(t, err) require.Equal(t, spec.Workflow, dbSpec.Workflow) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + _, err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) }) diff --git a/core/services/workflows/artifacts/v2/store.go b/core/services/workflows/artifacts/v2/store.go index 0eeb694cc6a..77c6e641495 100644 --- a/core/services/workflows/artifacts/v2/store.go +++ b/core/services/workflows/artifacts/v2/store.go @@ -3,7 +3,6 @@ package v2 import ( "context" "crypto/sha256" - "database/sql" "encoding/base64" "encoding/hex" "errors" @@ -234,22 +233,24 @@ func (h *Store) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.Wo return spec, err } +// GetWorkflowSpecList returns the persisted workflow specs (identity columns +// only). It backs the metering snapshot path. +func (h *Store) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.orm.GetWorkflowSpecList(ctx) +} + func (h *Store) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { return h.orm.UpsertWorkflowSpec(ctx, spec) } -// DeleteWorkflowArtifacts removes the workflow spec from the database. If not found, returns nil. -func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { - err := h.orm.DeleteWorkflowSpec(ctx, workflowID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - h.lggr.Warnw("failed to delete workflow spec: not found", "workflowID", workflowID) - return nil - } - return fmt.Errorf("failed to delete workflow spec: %w", err) - } +// DeleteWorkflowArtifacts removes the workflow spec from the database. If not +// found, returns (nil, nil). +func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { + return h.orm.DeleteWorkflowSpec(ctx, workflowID) +} - return nil +func (h *Store) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return h.orm.PauseWorkflowSpec(ctx, workflowID) } func (h *Store) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { diff --git a/core/services/workflows/artifacts/v2/store_test.go b/core/services/workflows/artifacts/v2/store_test.go index 59117a3a502..62ff154d520 100644 --- a/core/services/workflows/artifacts/v2/store_test.go +++ b/core/services/workflows/artifacts/v2/store_test.go @@ -87,7 +87,7 @@ func Test_Store_DeleteWorkflowArtifacts(t *testing.T) { require.NoError(t, err) // Delete the workflow artifacts by ID - err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) + _, err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) require.NoError(t, err) // Check that the workflow no longer exists diff --git a/core/services/workflows/syncer/v2/engine_registry.go b/core/services/workflows/syncer/v2/engine_registry.go index d1b3c90b938..82eba6d951d 100644 --- a/core/services/workflows/syncer/v2/engine_registry.go +++ b/core/services/workflows/syncer/v2/engine_registry.go @@ -23,7 +23,7 @@ type ServiceWithMetadata struct { services.Service } -// engineEntry holds the engine and its associated source for internal storage +// engineEntry holds the engine and its associated source for internal storage. type engineEntry struct { engine services.Service source string diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index e21a6cfff6c..1ff2776787e 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -2,12 +2,15 @@ package v2 import ( "context" + "database/sql" "encoding/hex" "errors" "fmt" "io" "maps" + "strconv" "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel" @@ -19,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -28,6 +32,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/workflows/dontime" generichost "github.com/smartcontractkit/chainlink-common/pkg/workflows/host" "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" @@ -64,6 +69,18 @@ type DrainableService interface { var ErrDrainInProgress = errors.New("drain in progress") +// Service-level metering identity constants for the workflow syncer. Service is +// the stable service constant; ResourcePool identifies the workflow_specs_v2 +// pool. The billing unit (ResourceType) and per-resource id (ResourceID) are +// carried on each Utilization. The coarse deployment/node/DON dimensions +// (product, tenant, environment, zone, don_id, node_id) are supplied at +// construction via WithIdentity. +const ( + meterService = "workflow-syncer-v2" + meterResourcePool = "workflow_specs_v2" + meterResourceType = "operations" +) + // eventHandler is a handler for WorkflowRegistryEvent events. Each event type has a corresponding method that handles the event. type eventHandler struct { services.Service @@ -88,8 +105,19 @@ type eventHandler struct { workflowEncryptionKey workflowkey.Key workflowDonSubscriber capabilities.DonSubscriber billingClient metering.BillingClient - orgResolver orgresolver.OrgResolver - secretsFetcher v2.SecretsFetcher + resourceManager *resourcemanager.ResourceManager + meterIdentity resourcemanager.ResourceIdentity + // resolvedDonID holds the workflow DON id once resolved from the don notifier + // at start (the engine runs on the workflow DON). It is resolved + // asynchronously so node boot is not blocked while waiting for the DON to be + // set, and read on the hot snapshot/emit paths; an atomic keeps that read + // lock-free. Nil until resolved; baseIdentity folds it into meterIdentity. + resolvedDonID atomic.Pointer[string] + // rmUnregister removes this handler from the ResourceManager's snapshot + // registry; set in start, called in close. Nil until started. + rmUnregister func() + orgResolver orgresolver.OrgResolver + secretsFetcher v2.SecretsFetcher // localSecretOverrides is keyed by owner address; values are secret id -> secret value localSecretOverrides map[string]map[string]string @@ -153,6 +181,27 @@ func WithBillingClient(client metering.BillingClient) func(*eventHandler) { } } +// WithResourceManager overrides the default (disabled) ResourceManager used to +// emit metering.v1.MeterRecord events for the workflow_specs_v2 storage lifecycle. +func WithResourceManager(rm *resourcemanager.ResourceManager) func(*eventHandler) { + return func(e *eventHandler) { + e.resourceManager = rm + } +} + +// WithIdentity supplies the coarse metering identity dimensions sourced once at +// construction in cre.go (product/tenant/environment/zone/node_id from node +// metering config; don_id resolved later by the handler). The syncer +// Service/ResourcePool are stable constants and always overwrite whatever the +// caller passes. +func WithIdentity(id resourcemanager.ResourceIdentity) func(*eventHandler) { + return func(e *eventHandler) { + id.Service = meterService + id.ResourcePool = meterResourcePool + e.meterIdentity = id + } +} + func WithShardExecutionGuard(client shardorchestrator.ClientInterface, shardingEnabled bool, shardID uint32) func(*eventHandler) { return func(e *eventHandler) { e.shardOrchestratorClient = client @@ -253,8 +302,10 @@ func WithModuleEngineVersion(v string) func(*eventHandler) { type WorkflowArtifactsStore interface { FetchWorkflowArtifacts(ctx context.Context, workflowID, binaryIdentifier, configIdentifier string) ([]byte, []byte, error) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) - DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error + DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + PauseWorkflowArtifacts(ctx context.Context, workflowID string) error DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error } @@ -306,7 +357,11 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode + tracer: noop.NewTracerProvider().Tracer(""), // default; can override in WithDebugMode + meterIdentity: resourcemanager.ResourceIdentity{ + Service: meterService, + ResourcePool: meterResourcePool, + }, } metricsInst, metricsErr := newMetrics() if metricsErr != nil { @@ -334,20 +389,70 @@ func NewEventHandler( return eh, nil } -func (h *eventHandler) start(_ context.Context) error { +func (h *eventHandler) start(ctx context.Context) error { if h.moduleLRU != nil { h.moduleLRU.Start() } + + if h.resourceManager != nil { + if err := h.resourceManager.Start(ctx); err != nil { + return fmt.Errorf("failed to start resource manager: %w", err) + } + // Register returns a func that unregisters. + h.rmUnregister = h.resourceManager.Register(h) + h.resolveWorkflowDonID() + } return nil } +// resolveWorkflowDonID asynchronously resolves the workflow DON id (the engine +// runs on the workflow DON) and folds it into the metering identity. It +// subscribes to the don notifier and stores the first DON's id, so node boot is +// never blocked waiting for the DON to be set. Until it resolves, emitted records +// carry an empty don_id (the host-injection fallback semantics); once resolved, +// every subsequent record and snapshot carries it. Resolution happens at most +// once per start. +func (h *eventHandler) resolveWorkflowDonID() { + if h.workflowDonSubscriber == nil || h.resolvedDonID.Load() != nil { + return + } + h.eng.Go(func(ctx context.Context) { + ch, unsubscribe, err := h.workflowDonSubscriber.Subscribe(ctx) + if err != nil { + h.lggr.Warnw("failed to subscribe to workflow DON for metering identity; don_id will be empty", "err", err) + return + } + defer unsubscribe() + select { + case <-ctx.Done(): + return + case don := <-ch: + donID := strconv.FormatUint(uint64(don.ID), 10) + h.resolvedDonID.Store(&donID) + h.lggr.Debugw("resolved workflow DON id for metering identity", "donID", donID) + } + }) +} + func (h *eventHandler) close() error { if h.moduleLRU != nil { h.moduleLRU.Close() } es := h.engineRegistry.PopAll() - cs := make([]io.Closer, 0, len(es)+1) + // No metering is emitted on close: meter records anchor on workflow-spec + // storage transitions, not engine lifecycle, so stopping an engine at + // shutdown leaves the persisted spec (and therefore its metered level) + // untouched. A spec that is genuinely released stops being snapshotted. + // Stop snapshotting this handler, then close the ResourceManager service. + if h.rmUnregister != nil { + h.rmUnregister() + h.rmUnregister = nil + } + cs := make([]io.Closer, 0, len(es)+2) cs = append(cs, h.engineLimiters) + if h.resourceManager != nil { + cs = append(cs, h.resourceManager) + } for _, e := range es { cs = append(cs, e) } @@ -444,7 +549,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { var err error defer func() { - if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, err); err2 != nil { + if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, customerFacingError(err)); err2 != nil { h.lggr.Errorf("failed to emit status changed event: %+v", err2) } }() @@ -469,21 +574,20 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { wfID := payload.WorkflowID.Hex() - // Get workflow spec from database to get owner and name info for organization lookup - // Alternative: wire through workflowOwner into the Event, but that requires a lot more surgery - spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID) + // Get workflow spec from database to get owner and name info for organization lookup, + // and to check if the spec exists to determine if a MeterRecord is warranted. var wfOwner, wfName, orgID string - if err != nil { - // Workflow spec not found, proceed with deletion but without event metadata - h.lggr.Warnw("Workflow spec not found during deletion, proceeding without org info", "workflowID", wfID, "error", err) - } else { + if spec, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID); gerr == nil && spec != nil { wfOwner = spec.WorkflowOwner wfName = spec.WorkflowName - if wfOwner != "" { - orgID, err = h.fetchOrganizationID(ctx, wfOwner) - if err != nil { - h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", err) - } + } else if gerr != nil && !errors.Is(gerr, sql.ErrNoRows) { + h.lggr.Errorw("failed to read workflow spec during deletion, proceeding without metadata", "workflowID", wfID, "error", gerr) + } + if wfOwner != "" { + if resolvedOrgID, orgErr := h.fetchOrganizationID(ctx, wfOwner); orgErr != nil { + h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", orgErr) + } else { + orgID = resolvedOrgID } } ctx = contexts.WithCRE(ctx, contexts.CRE{Org: orgID, Owner: wfOwner, Workflow: wfID}) @@ -507,7 +611,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { } }() - if herr = h.workflowDeletedEvent(ctx, payload); herr != nil { + if herr = h.workflowDeletedEvent(ctx, payload, wfOwner); herr != nil { if errors.Is(herr, ErrDrainInProgress) { logCustMsg(ctx, cma, fmt.Sprintf("workflow deletion deferred: %v", herr), h.lggr) } else { @@ -559,13 +663,18 @@ func (h *eventHandler) workflowRegisteredEvent( // - existing registration that has been updated with a new status spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, payload.WorkflowID.Hex()) switch { - case err != nil: + case errors.Is(err, sql.ErrNoRows): newSpec, innerErr := h.createWorkflowSpec(ctx, payload) if innerErr != nil { return innerErr } + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) + spec = newSpec + case err != nil: + return fmt.Errorf("failed to get workflow spec: %w", err) case spec.WorkflowID != payload.WorkflowID.Hex() || spec.WorkflowOwner != hex.EncodeToString(payload.WorkflowOwner) || spec.WorkflowName != payload.WorkflowName: @@ -573,12 +682,38 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } + // A different spec's artifacts were persisted under this key: the newly + // stored spec is a fresh durable resource, so emit a +1 delta. + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) spec = newSpec case spec.Status != status: - spec.Status = status - if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { - return fmt.Errorf("failed to update workflow spec: %w", innerErr) + if status == job.WorkflowSpecStatusActive && spec.Workflow == "" { + // Activating a paused tombstone: no delta. + newSpec, innerErr := h.createWorkflowSpec(ctx, payload) + if innerErr != nil { + return innerErr + } + spec = newSpec + } else { + spec.Status = status + if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { + spec.RegisteredAt = int64(payload.CreatedAt) + } + if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { + return fmt.Errorf("failed to update workflow spec: %w", innerErr) + } + // Status-only flip: no artifact-persistence transition, no delta. + } + } + + // Legacy-row convergence: backfill registered_at for pre-migration rows + // (at most once per row). Fail-open: a write error is logged, not returned. + if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { + spec.RegisteredAt = int64(payload.CreatedAt) + if _, err := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); err != nil { + h.lggr.Warnw("failed to backfill registered_at", "workflowID", spec.WorkflowID, "err", err) } } @@ -670,6 +805,7 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, Attributes: payload.Attributes, + RegisteredAt: int64(payload.CreatedAt), } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -824,32 +960,17 @@ func (h *eventHandler) createEngineModule( return engineModule } -// workflowPausedEvent handles the WorkflowPausedEvent event type. This method must remain idempotent. -func (h *eventHandler) workflowPausedEvent( - ctx context.Context, - payload WorkflowPausedEvent, -) error { - return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}) -} - -// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. -func (h *eventHandler) workflowDeletedEvent( - ctx context.Context, - payload WorkflowDeletedEvent, -) error { - // The order in the handler is slightly different to the order in `tryEngineCleanup`. - // This is because the engine requires its corresponding DB record to be present to be successfully - // closed. - // At the same time, popping the engine should occur last to allow deletes to be retried if any of the - // prior steps fail. - workflowID := payload.WorkflowID.Hex() - e, ok := h.engineRegistry.Get(payload.WorkflowID) +// stopEngine drains (returning ErrDrainInProgress while executions remain) and +// closes the engine for workflowID if one is registered. +// Returns the drainable handle (nil-able) for drain-completed metrics. +func (h *eventHandler) stopEngine(ctx context.Context, workflowID types.WorkflowID) (DrainableService, error) { + e, ok := h.engineRegistry.Get(workflowID) var drainable DrainableService - var isDrainable bool if ok { + var isDrainable bool if drainable, isDrainable = e.Service.(DrainableService); isDrainable { if started := drainable.Drain(); started { - h.lggr.Infow("initiated drain for workflow engine", "workflowID", workflowID) + h.lggr.Infow("initiated drain for workflow in workflow engine", "workflowID", workflowID.String()) if h.metrics != nil { h.metrics.incrementDrainStarted(ctx) } @@ -860,32 +981,92 @@ func (h *eventHandler) workflowDeletedEvent( h.metrics.incrementDeleteDeferred(ctx, "drain_in_progress") } h.lggr.Infow("workflow deletion deferred: active executions still running", - "workflowID", workflowID, + "workflowID", workflowID.String(), "activeExecutions", active) - return fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) + return nil, fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) } } if innerErr := e.Close(); innerErr != nil && !errors.Is(innerErr, services.ErrAlreadyStopped) { - return fmt.Errorf("failed to close workflow engine: %w", innerErr) + return nil, fmt.Errorf("failed to close workflow engine: %w", innerErr) } } + return drainable, nil +} - if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { +// releaseSpecStorage deletes the persisted spec row and, iff a row was +// actually removed, emits the -1 generation delta. RowsAffected is the +// exactly-once gate: redeliveries observe no row and emit nothing; a transient +// DELETE error returns before emission so the retry emits when the delete +// lands. The event_id is derived AFTER the delete from the row's persisted +// registered_at, so all nodes emit the identical id. Module-cache cleanup +// always runs. owner may be empty; org resolution is fail-open. +func (h *eventHandler) releaseSpecStorage(ctx context.Context, workflowID, owner string) error { + deletedSpec, err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, workflowID) + if err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } + if deletedSpec != nil { + parts := []string{workflowID} + if deletedSpec.RegisteredAt > 0 { + parts = append(parts, strconv.FormatInt(deletedSpec.RegisteredAt, 10)) + } + h.emitSpecDelta(ctx, -1, workflowID, owner, resourcemanager.EventID("workflow-spec-delete", parts...)) + } + h.cleanupModuleCache(workflowID) + return nil +} - h.cleanupModuleCache(payload.WorkflowID.Hex()) +// workflowPausedEvent handles WorkflowPaused. Idempotent. +// +// Pause is level-neutral for metering: the spec row survives as a tombstone +// (status=paused, artifact payload cleared to free storage) so the +// registration generation stays metered until deletion — mirroring the +// on-chain registry, which retains the paused registration. Emitting per-cycle +// pause/activate deltas is impossible without drift: no DON-consistent +// per-occurrence discriminator exists, so a -1 here would force the +// re-activation +1 to reuse the register event_id and be dropped by consumer +// dedup. Deltas therefore fire only at generation boundaries (register/delete). +func (h *eventHandler) workflowPausedEvent( + ctx context.Context, + payload WorkflowPausedEvent, +) error { + workflowID := payload.WorkflowID.Hex() + if _, err := h.stopEngine(ctx, payload.WorkflowID); err != nil { + return err + } + if err := h.workflowArtifactsStore.PauseWorkflowArtifacts(ctx, workflowID); err != nil { + return fmt.Errorf("failed to pause workflow artifacts: %w", err) + } + h.cleanupModuleCache(workflowID) + if _, err := h.engineRegistry.Pop(payload.WorkflowID); err != nil && !errors.Is(err, ErrNotFound) { + return err + } + return nil +} - _, err := h.engineRegistry.Pop(payload.WorkflowID) +// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. +func (h *eventHandler) workflowDeletedEvent( + ctx context.Context, + payload WorkflowDeletedEvent, + owner string, +) error { + workflowID := payload.WorkflowID.Hex() + drainable, err := h.stopEngine(ctx, payload.WorkflowID) + if err != nil { + return err + } + if err := h.releaseSpecStorage(ctx, workflowID, owner); err != nil { + return err + } + _, err = h.engineRegistry.Pop(payload.WorkflowID) if errors.Is(err, ErrNotFound) { return nil } if err != nil { return err } - - if isDrainable { + if drainable != nil { startedAt, exists := drainable.DrainStartedAt() if exists && h.metrics != nil { h.metrics.recordDrainCompleted(ctx, time.Since(startedAt)) @@ -894,6 +1075,120 @@ func (h *eventHandler) workflowDeletedEvent( return nil } +// ReleaseOrphanedSpec implements the workflow registry's downtime-orphan sweep +// contract: release the persisted spec for a workflow absent from all sources' +// metadata with no running engine. +func (h *eventHandler) ReleaseOrphanedSpec(ctx context.Context, workflowID, owner string) error { + return h.releaseSpecStorage(ctx, workflowID, owner) +} + +// emitSpecDelta emits one metering.v1.MeterRecord (METER_ACTION_UPDATE) +// capturing a signed ±delta to the durable workflow_specs_v2 level for +// workflowID. Each record is a first-class billing event: a +1 when a new +// spec's artifacts are persisted, a -1 when a spec is deleted. It is the single +// emission helper for every syncer meter record; callers invoke it only after +// the workflow-spec storage mutation has committed. +// +// owner is the workflow owner as stored in durable state; the organization is +// resolved fail-open at emit time and is never persisted. +// Emission is fail-open and returns no error, so metering can never gate, +// delay, or fail the storage operation it records. +// +// eventID is the caller-supplied, deterministic cross-node dedup key. The syncer +// is a workflow-DON service driven by reconciliation against the shared on-chain +// WorkflowRegistry, so every workflow-DON node sees the same reconciliation +// event and derives the identical id from it (see the call sites). It is never +// generated here; an empty eventID is rejected by the ResourceManager. +func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowID, owner, eventID string) { + if h.resourceManager == nil { + return + } + var orgID string + if h.orgResolver != nil && owner != "" { + if resolved, err := h.orgResolver.Get(ctx, owner); err != nil { + h.lggr.Warnw("failed to resolve org ID for metering", "owner", owner, "err", err) + } else { + orgID = resolved + } + } + // resource_id = workflow_id (the syncer meters one durable spec per workflow). + h.resourceManager.EmitDelta(ctx, h.baseIdentity(), eventID, delta, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }) +} + +// baseIdentity returns the handler's metering identity with the workflow DON id +// folded in once it has been resolved from the don notifier. meterIdentity itself +// is immutable after construction (set via WithIdentity); only the DON id is +// learned later, so it is read from an atomic and overlaid here. +func (h *eventHandler) baseIdentity() resourcemanager.ResourceIdentity { + id := h.meterIdentity + if donID := h.resolvedDonID.Load(); donID != nil { + nodeID := "" + if id.Don != nil { + nodeID = id.Don.NodeID + } + id.Don = &resourcemanager.DonIdentity{ + DonID: *donID, + NodeID: nodeID, + } + } + return id +} + +// ResourceIdentity implements resourcemanager.Meterable: it returns the syncer's +// base identity (coarse deployment/DON/node dimensions + service/resource_pool). +// The ResourceManager uses it as the top-level identity of each emitted +// Snapshot. +func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { + return h.baseIdentity() +} + +// GetWorkflowSpecList backs the orphan sweep and the metering snapshot path. +func (h *eventHandler) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.workflowArtifactsStore.GetWorkflowSpecList(ctx) +} + +// GetUtilization implements resourcemanager.Meterable: it returns one +// SnapshotEntry per persisted workflow_specs_v2 spec. +// The durable resource is the stored spec, NOT the running engine. +// +// resource_id is the workflow_id; the organization is resolved fail-open from +// the spec's stored owner through the org resolver. Must fail open. +func (h *eventHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + specs, err := h.workflowArtifactsStore.GetWorkflowSpecList(ctx) + if err != nil { + h.lggr.Warnw("failed to list persisted workflow specs for metering snapshot; skipping tick", "err", err) + return nil + } + base := h.baseIdentity() + entries := make([]resourcemanager.SnapshotEntry, 0, len(specs)) + for _, spec := range specs { + if spec == nil { + continue + } + var orgID string + if h.orgResolver != nil && spec.WorkflowOwner != "" { + if resolved, err := h.orgResolver.Get(ctx, spec.WorkflowOwner); err == nil { + orgID = resolved + } + } + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: base, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: spec.WorkflowID, + OrgID: orgID, + }), + }, + }) + } + return entries +} + // tryEngineCleanup attempts to stop the workflow engine for the given workflow ID. Does nothing if the // workflow engine is not running. func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go new file mode 100644 index 00000000000..f58f460c439 --- /dev/null +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -0,0 +1,802 @@ +package v2 + +import ( + "context" + "errors" + "math/big" + "strconv" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + commonlogger "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + + "github.com/smartcontractkit/chainlink/v2/core/capabilities" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" + workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// meteringwfID is the workflow ID that GenerateWorkflowID produces from the +// stub store's FetchWorkflowArtifacts return values (binary="binary", +// config="config") with owner 0xaabbccdd and name "wf-name". Tests that use +// persistUpserts and register as Active must use this ID so tryEngineCreate's +// validation passes. +var meteringwfID = func() types.WorkflowID { + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + wfIDBytes, _ := pkgworkflows.GenerateWorkflowID(owner, "wf-name", []byte("binary"), []byte("config"), "") + return types.WorkflowID(wfIDBytes) +}() + +// recordingEmitter is a fake resourcemanager.Emitter that decodes and stores +// every emitted MeterRecord. If err is set, Emit fails instead. +type recordingEmitter struct { + mu sync.Mutex + err error + records []*meteringpb.MeterRecord +} + +func (r *recordingEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.err != nil { + return r.err + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + r.records = append(r.records, &record) + return nil +} + +func (r *recordingEmitter) Records() []*meteringpb.MeterRecord { + r.mu.Lock() + defer r.mu.Unlock() + records := make([]*meteringpb.MeterRecord, len(r.records)) + copy(records, r.records) + return records +} + +func newMeteringResourceManager(t *testing.T, enabled bool, emitter resourcemanager.Emitter) *resourcemanager.ResourceManager { + t.Helper() + return resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: enabled, + Emitter: emitter, + }) +} + +// recordingSnapshotEmitter is a fake resourcemanager.Emitter that decodes and +// stores every emitted MeterSnapshot (one per active resource). Used to drive +// the Meterable snapshot path. +type recordingSnapshotEmitter struct { + mu sync.Mutex + snapshots []*meteringpb.MeterSnapshot +} + +func (r *recordingSnapshotEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + r.snapshots = append(r.snapshots, &snapshot) + return nil +} + +func (r *recordingSnapshotEmitter) Snapshots() []*meteringpb.MeterSnapshot { + r.mu.Lock() + defer r.mu.Unlock() + snapshots := make([]*meteringpb.MeterSnapshot, len(r.snapshots)) + copy(snapshots, r.snapshots) + return snapshots +} + +func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + &confidentialrelay.ExecutionHandlers{}, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithResourceManager(rm), + WithEngineFactoryFn(mockEngineFactory), + ) + require.NoError(t, err) + return h +} + +// requireSpecDelta asserts that record is a workflow-syncer-v2 spec delta: a +// METER_ACTION_UPDATE carrying a single utilization with the given signed value +// and resource_id (= workflow_id). event_id must be the deterministic, +// cross-node-identical id wantEventID (an opaque string that is never +// format-validated). +func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workflowID, wantEventID string) { + t.Helper() + require.NotNil(t, record.Identity) + assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) + assert.Equal(t, "workflow_specs_v2", record.Identity.ResourcePool) + // Every syncer meter record is a signed level delta (UPDATE), never a + // RESERVE/RELEASE lifecycle edge. + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.Action) + assert.NotNil(t, record.Timestamp) + require.Len(t, record.Utilizations, 1) + util := record.Utilizations[0] + assert.Equal(t, value, util.Value) + assert.Equal(t, "operations", util.ResourceType) + // resource_id = workflow_id for the syncer (no shared physical resource). + assert.Equal(t, workflowID, util.ResourceId) + // event_id is the deterministic reconciliation-derived id, identical on every + // workflow-DON node; it is an opaque string with no validated format. + assert.Equal(t, wantEventID, util.EventId) +} + +func Test_meterRecords(t *testing.T) { + t.Parallel() + + wfOwner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + + t.Run("registered event persisting a new spec emits +1", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{1} + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.NoError(t, err) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), "0")) + }) + + t.Run("reprocessed registered event emits the IDENTICAL event_id (cross-node dedup)", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + // The stub never returns a stored spec, so each call replays the + // new-spec path exactly as a reprocessed event (or a second node + // reconciling the same on-chain event) would. + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{2}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + CreatedAt: 123, + } + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) + + records := emitter.Records() + require.Len(t, records, 2) + require.Len(t, records[0].Utilizations, 1) + require.Len(t, records[1].Utilizations, 1) + // The same reconciliation event MUST yield the identical event_id, so the + // billing consumer dedups reprocessing and cross-node duplicates. It is + // derived deterministically from the on-chain workflowID + CreatedAt. + want := resourcemanager.EventID("workflow-spec-register", types.WorkflowID{2}.Hex(), "123") + assert.Equal(t, want, records[0].Utilizations[0].GetEventId()) + assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) + }) + + t.Run("activating an already-stored spec emits nothing (status-only update)", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{3} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + WorkflowName: "wf-name", + }, + }, newMeteringResourceManager(t, true, emitter)) + + // Same workflow ID already stored, only the status changes: the spec + // stays stored, so no artifact-persistence transition and no delta. + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.NoError(t, err) + + assert.Empty(t, emitter.Records()) + }) + + t.Run("paused event emits nothing", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{4} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + + // Pause routes through the delete path, but the spec's release is + // realized by its absence from subsequent snapshots, not a delta. + assert.Empty(t, emitter.Records()) + }) + + t.Run("deleted event emits -1 after artifacts are deleted", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{5} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) + }) + + t.Run("no record when persisting a new spec fails", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{upsertErr: assert.AnError}, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{6}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record when deleting artifacts fails", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{7} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + deleteErr: assert.AnError, + }, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record while a delete is deferred by drain; exactly one on the successful retry", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{8} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + drainable := &mockDrainableEngine{} + drainable.activeExecutions.Store(1) + require.NoError(t, h.engineRegistry.Add(wfID, "test-source", drainable)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, ErrDrainInProgress) + assert.Empty(t, emitter.Records()) + + drainable.activeExecutions.Store(0) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) + }) + + t.Run("emit failure never fails event handling", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{9} + emitter := &recordingEmitter{err: errors.New("beholder unavailable")} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + })) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + assert.Empty(t, emitter.Records()) + }) + + t.Run("disabled resource manager emits nothing", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, false, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{10}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + })) + assert.Empty(t, emitter.Records()) + }) + + t.Run("snapshot emits one MeterSnapshot per persisted spec", func(t *testing.T) { + t.Parallel() + emitter := &recordingSnapshotEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + rm := resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + + // The snapshot enumerates persisted specs from the store, not running + // engines, so paused-but-stored specs are still accounted for. + wfID1 := types.WorkflowID{20} + wfID2 := types.WorkflowID{21} + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{ + {WorkflowID: wfID1.Hex(), WorkflowOwner: "aabbccdd"}, + {WorkflowID: wfID2.Hex(), WorkflowOwner: "aabbccdd"}, + }, + } + h := newMeteringTestHandler(t, store, rm) + unregister := rm.Register(h) + t.Cleanup(unregister) + + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.Snapshots()) == 2 + }, time.Second, time.Millisecond) + snapshots := emitter.Snapshots() + require.Len(t, snapshots, 2) + + byWorkflowID := map[string]*meteringpb.MeterSnapshot{} + for _, snap := range snapshots { + require.NotNil(t, snap.Identity) + assert.Equal(t, "workflow-syncer-v2", snap.Identity.Service) + assert.Equal(t, "workflow_specs_v2", snap.Identity.ResourcePool) + require.Len(t, snap.Utilization, 1) + assert.Equal(t, "1", snap.Utilization[0].Value) + // resource_id = workflow_id fully identifies the resource; no labels. + byWorkflowID[snap.Utilization[0].ResourceId] = snap + } + require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first persisted spec") + require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second persisted spec") + }) +} + +// fakeOrgResolver is a no-network OrgResolver that maps workflow owners (hex) +// to organization IDs. It asserts that metering org resolution is exercised +// (previously org was entirely untested). +type fakeOrgResolver struct { + orgs map[string]string +} + +func (f *fakeOrgResolver) Get(_ context.Context, owner string) (string, error) { + if org, ok := f.orgs[owner]; ok { + return org, nil + } + return "", errors.New("owner not found") +} + +func (f *fakeOrgResolver) Start(context.Context) error { return nil } +func (f *fakeOrgResolver) Close() error { return nil } +func (f *fakeOrgResolver) HealthReport() map[string]error { return nil } +func (f *fakeOrgResolver) Ready() error { return nil } +func (f *fakeOrgResolver) Name() string { return "fake-org-resolver" } + +// newMeteringTestHandlerWithOrg is newMeteringTestHandler plus a wired +// OrgResolver, so tests can assert OrgId on emitted records. +func newMeteringTestHandlerWithOrg(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager, org orgresolver.OrgResolver) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + &confidentialrelay.ExecutionHandlers{}, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithResourceManager(rm), + WithEngineFactoryFn(mockEngineFactory), + WithOrgResolver(org), + ) + require.NoError(t, err) + return h +} + +// redeliveredDeleteSpecs returns a fresh stub seeded with a stored spec and the +// owner the metering tests use, so a delete + redelivered-delete pair can be +// driven through Handle (which pre-reads the spec itself). +func redeliveredDeleteStore() *stubWorkflowArtifactsStore { + return &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{5}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + } +} + +// a redelivered delete (after the spec was already removed) must NOT emit a +// second -1. DeleteWorkflowArtifacts maps ErrNoRows→nil, so without gating on +// the pre-read the redelivery would double-emit. +func Test_meterRecords_RedeliveredDeleteEmitsExactlyOneDelta(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{5} + emitter := &recordingEmitter{} + store := redeliveredDeleteStore() + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + deleteEvent := Event{Name: WorkflowDeleted, Data: WorkflowDeletedEvent{WorkflowID: wfID}} + require.NoError(t, h.Handle(t.Context(), deleteEvent)) + require.NoError(t, h.Handle(t.Context(), deleteEvent)) // redelivery + + records := emitter.Records() + require.Len(t, records, 1, "redelivered delete must not emit a second -1") + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) +} + +// Pause and activate are level-neutral: neither emits a metering delta. +// The +1 from register stays as the sole record through a pause→activate cycle. +func Test_meterRecords_PauseActivateCycleIsLevelNeutral(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(123) + wantEventID := resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10)) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.Len(t, emitter.Records(), 1) + requireSpecDelta(t, emitter.Records()[0], "1", wfID.Hex(), wantEventID) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.Len(t, emitter.Records(), 1, "pause must not emit a delta") + + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.Len(t, emitter.Records(), 1, "activate-after-pause must not emit a delta") +} + +// OrgId is resolved from the workflow owner and stamped on emitted records. +func Test_meterRecords_OrgIdResolvedOnRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + orgR := &fakeOrgResolver{orgs: map[string]string{"aabbccdd": "org-42"}} + h := newMeteringTestHandlerWithOrg(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter), orgR) + + wfID := types.WorkflowID{50} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + + records := emitter.Records() + require.Len(t, records, 1) + require.Len(t, records[0].Utilizations, 1) + assert.Equal(t, "org-42", records[0].Utilizations[0].OrgId, "OrgId must be resolved and stamped on the record") +} + +// don_id is folded into the metering identity once resolved and stamped on both +// emitted records and snapshot entries. +func Test_meterRecords_DonIDOnRecordAndSnapshot(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + wfID := types.WorkflowID{60} + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{{WorkflowID: wfID.Hex(), WorkflowOwner: "aabbccdd"}}, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + // Simulate a resolved workflow DON id (as resolveWorkflowDonID would store). + resolvedDon := "7" + h.resolvedDonID.Store(&resolvedDon) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + records := emitter.Records() + require.Len(t, records, 1) + require.NotNil(t, records[0].Identity.GetDon()) + assert.Equal(t, "7", records[0].Identity.GetDon().GetDonId(), "record identity must carry resolved don_id") + + entries := h.GetUtilization(t.Context()) + require.Len(t, entries, 1) + require.NotNil(t, entries[0].Identity.Don) + assert.Equal(t, "7", entries[0].Identity.Don.DonID, "snapshot identity must carry resolved don_id") +} + +// a transient DB error on GetWorkflowSpec (not a missing row) must surface +// an error and emit no +1, so the event is retried instead of creating a +// spurious spec + delta. +func Test_meterRecords_TransientDBErrorOnGetSpecEmitsNoDelta(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{getSpecErr: errors.New("db connection lost")}, + newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{70}, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + }) + require.Error(t, err) + assert.Empty(t, emitter.Records(), "transient DB error must not take the new-spec path or emit a +1") +} + +// A full register→pause→activate→delete cycle emits exactly two records: +// one +1 at register and one -1 at delete. Pause and activate emit nothing. +// The delete event_id carries the registered_at that flowed insert→RETURNING, +// proving the generation-scoped id is DON-consistent. +func Test_meterRecords_FullLifecycleEmitsExactlyTwoRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(123) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly two records: +1 at register, -1 at delete") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} + +// A transient delete error returns before emission (no -1); a successful retry +// emits exactly one -1. This fixes the lost-−1 hazard from the old pre-read gate. +func Test_meterRecords_TransientDeleteErrorThenRetryEmitsOnce(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + deleteErr: assert.AnError, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{77} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + require.Len(t, emitter.Records(), 1, "register emits +1") + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()[1:], "failed delete must not emit a -1") + + store.deleteErr = nil + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly one -1 after successful retry") + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), "1")) +} + +// A legacy row with RegisteredAt == 0 produces a delete event_id with no +// timestamp part (the fallback for pre-migration rows). +func Test_meterRecords_LegacyRowDeleteUsesFallbackID(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{88}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + RegisteredAt: 0, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: types.WorkflowID{88}}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", types.WorkflowID{88}.Hex(), + resourcemanager.EventID("workflow-spec-delete", types.WorkflowID{88}.Hex())) +} + +// Identical event sequences with RM nil / disabled / erroring emitter produce +// identical row+engine outcomes and zero handler errors (fail-open equivalence). +func Test_meterRecords_FailOpenEquivalence(t *testing.T) { + t.Parallel() + wfID := meteringwfID + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: 1, + } + delEvt := WorkflowDeletedEvent{WorkflowID: wfID} + + runCycle := func(t *testing.T, rm *resourcemanager.ResourceManager, emitter resourcemanager.Emitter) (*stubWorkflowArtifactsStore, *eventHandler) { + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, rm) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), delEvt, "aabbccdd")) + return store, h + } + + // RM nil + nilStore, nilH := runCycle(t, nil, nil) + assert.Nil(t, nilH.resourceManager) + assert.Nil(t, nilStore.spec, "spec should be deleted by the cycle") + + // RM disabled + disabledEmitter := &recordingEmitter{} + disabledRM := newMeteringResourceManager(t, false, disabledEmitter) + disabledStore, disabledH := runCycle(t, disabledRM, disabledEmitter) + assert.NotNil(t, disabledH.resourceManager) + assert.Empty(t, disabledEmitter.Records(), "disabled RM emits nothing") + assert.Nil(t, disabledStore.spec, "spec should be deleted by the cycle") + + // RM enabled but erroring emitter + errEmitter := &recordingEmitter{err: errors.New("beholder unavailable")} + errRM := newMeteringResourceManager(t, true, errEmitter) + errStore, errH := runCycle(t, errRM, errEmitter) + assert.NotNil(t, errH.resourceManager) + assert.Empty(t, errEmitter.Records(), "erroring emitter stores nothing") + assert.Nil(t, errStore.spec, "spec should be deleted by the cycle") +} + +// The orphan sweep releases a paused tombstone (no engine) with exactly one -1 +// carrying the generation delete-id. This is the sweep's new load-bearing case: +// workflows deleted on-chain while paused have a tombstone but no engine. +func Test_meterRecords_SweepReleasesPausedTombstone(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(456) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + })) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + // After pause the engine is popped; the tombstone has no engine → sweep path. + require.NoError(t, h.ReleaseOrphanedSpec(t.Context(), wfID.Hex(), "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "register +1 and sweep -1") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 24ae08d7bd4..fcf1f36c551 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -2,12 +2,14 @@ package v2 import ( "context" + "database/sql" "encoding/base64" "encoding/hex" "errors" "fmt" "math/big" "net" + "strconv" "sync/atomic" "testing" "time" @@ -38,6 +40,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder/beholdertest" "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -1020,13 +1023,21 @@ func (m *mockArtifactStore) UpsertWorkflowSpec(ctx context.Context, spec *job.Wo return m.artifactStore.UpsertWorkflowSpec(ctx, spec) } -func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { +func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { if m.deleteWorkflowArtifactsErr != nil { - return m.deleteWorkflowArtifactsErr + return nil, m.deleteWorkflowArtifactsErr } return m.artifactStore.DeleteWorkflowArtifacts(ctx, workflowID) } +func (m *mockArtifactStore) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return m.artifactStore.PauseWorkflowArtifacts(ctx, workflowID) +} + +func (m *mockArtifactStore) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return m.artifactStore.GetWorkflowSpecList(ctx) +} + func (m *mockArtifactStore) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { return m.artifactStore.DeleteWorkflowArtifactsBatch(ctx, workflowIDs) } @@ -1131,7 +1142,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1186,7 +1197,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1287,7 +1298,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.Error(t, err, failWith) // Verify the record is still in the DB @@ -1301,29 +1312,72 @@ func Test_workflowDeletedHandler(t *testing.T) { } type stubWorkflowArtifactsStore struct { - spec *job.WorkflowSpec - deleteErr error - deleteCalls atomic.Int32 + spec *job.WorkflowSpec + specs []*job.WorkflowSpec + persistUpserts bool + upsertErr error + deleteErr error + listErr error + getSpecErr error + deleteCalls atomic.Int32 + pauseCalls atomic.Int32 + fetchCalls atomic.Int32 } func (s *stubWorkflowArtifactsStore) FetchWorkflowArtifacts(context.Context, string, string, string) ([]byte, []byte, error) { - return nil, nil, nil + s.fetchCalls.Add(1) + return []byte("binary"), []byte("config"), nil } func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (*job.WorkflowSpec, error) { + if s.getSpecErr != nil { + return nil, s.getSpecErr + } if s.spec == nil { - return nil, errors.New("not found") + return nil, sql.ErrNoRows } return s.spec, nil } -func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(context.Context, *job.WorkflowSpec) (int64, error) { +func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(_ context.Context, spec *job.WorkflowSpec) (int64, error) { + if s.upsertErr != nil { + return 0, s.upsertErr + } + if s.persistUpserts { + cp := *spec + s.spec = &cp + } return 1, nil } -func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) error { +func (s *stubWorkflowArtifactsStore) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + if s.listErr != nil { + return nil, s.listErr + } + return s.specs, nil +} + +func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) (*job.WorkflowSpec, error) { s.deleteCalls.Add(1) - return s.deleteErr + if s.deleteErr != nil { + return nil, s.deleteErr + } + if s.spec == nil { + return nil, nil + } + deleted := *s.spec + s.spec = nil + return &deleted, nil +} + +func (s *stubWorkflowArtifactsStore) PauseWorkflowArtifacts(context.Context, string) error { + s.pauseCalls.Add(1) + if s.spec != nil { + s.spec.Status = job.WorkflowSpecStatusPaused + s.spec.Workflow = "" + s.spec.Config = "" + } + return nil } func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifactsBatch(context.Context, []string) error { @@ -1346,7 +1400,7 @@ func Test_workflowDeletedEvent_DrainInProgress(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.Error(t, err) require.ErrorIs(t, err, ErrDrainInProgress) assert.Equal(t, int32(1), drainable.drainCalls.Load()) @@ -1372,7 +1426,7 @@ func Test_workflowDeletedEvent_IgnoresErrAlreadyStopped(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.NoError(t, err) assert.Equal(t, int32(1), drainable.closeCalls.Load()) assert.Equal(t, int32(1), artifactStore.deleteCalls.Load()) @@ -1616,7 +1670,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold }) // Mock ORM responses - mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, errors.New("not found")) + mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, sql.ErrNoRows) mockORM.EXPECT().UpsertWorkflowSpec(mock.Anything, mock.AnythingOfType("*job.WorkflowSpec")).Return(int64(1), nil) // Set up handler @@ -1712,7 +1766,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold } mockDeleteORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(spec, nil) - mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil) + mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(&job.WorkflowSpec{}, nil) deleteArtifactStore, err := artifacts.NewStore(lggr, mockDeleteORM, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ ArtifactStorageHost: "example.com", @@ -1880,3 +1934,214 @@ func (c *confidentialCap) Initialise(_ context.Context, _ core.StandardCapabilit } var _ server.ClientCapability = &confidentialCap{} + +// assertStubState checks the in-memory stub's spec state. +func assertStubState(t *testing.T, s *stubWorkflowArtifactsStore, exists bool, status job.WorkflowSpecStatus, artifactsEmpty bool, registeredAt int64) { + t.Helper() + if !exists { + assert.Nil(t, s.spec, "spec should not exist") + return + } + require.NotNil(t, s.spec, "spec should exist") + assert.Equal(t, status, s.spec.Status) + if artifactsEmpty { + assert.Empty(t, s.spec.Workflow, "workflow artifacts should be empty") + assert.Empty(t, s.spec.Config, "config artifacts should be empty") + } else { + assert.NotEmpty(t, s.spec.Workflow, "workflow artifacts should be present") + } + assert.Equal(t, registeredAt, s.spec.RegisteredAt) +} + +// Test_specStorage_StateMachine exercises the storage state machine across +// (row state × event) transitions, verifying row existence, Status, artifact +// presence, RegisteredAt, engine-registry contents, and fetch call counts. +func Test_specStorage_StateMachine(t *testing.T) { + t.Parallel() + + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + hexWorkflow := hex.EncodeToString(binaryData) + createdAt := uint64(999) + + makeHandler := func(store *stubWorkflowArtifactsStore) *eventHandler { + return newMeteringTestHandler(t, store, nil) + } + activePayload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + t.Run("absent + Activated → fetch, insert, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, int32(1), store.fetchCalls.Load(), "exactly one fetch") + }) + + t.Run("active + Paused → tombstone, engine popped; redelivery no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be popped") + // Redelivery + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + }) + + t.Run("tombstone + Activated → fetch, full restore, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + fetchBefore := store.fetchCalls.Load() + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, fetchBefore+1, store.fetchCalls.Load(), "exactly one new fetch for restore") + }) + + t.Run("tombstone + Deleted → row removed, engine steps no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be absent") + }) + + t.Run("paused+artifacts + Activated → status flip, no fetch", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusPaused, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: int64(createdAt), + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + }) + + t.Run("absent + Paused → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("absent + Deleted → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "")) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("active RegisteredAt==0 + Activated (no status change) → backfill", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: 0, + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + }) +} + +// Test_handler_SourceParity_PauseActivateCycle runs a full register→pause→ +// activate→delete cycle twice with contract-style and gRPC-style source +// identifiers, asserting identical storage outcomes, engine-registry source +// scoping, and identical metering records/ids. +func Test_handler_SourceParity_PauseActivateCycle(t *testing.T) { + t.Parallel() + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + createdAt := uint64(777) + sources := []string{ + "contract:42:0xabcdef1234567890", + "grpc:centralized-registry:v1", + } + + for _, source := range sources { + t.Run(source, func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + Source: source, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + // Verify engine is registered with the correct source + entry, ok := h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID, Source: source})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + entry, ok = h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID, Source: source}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + + // Both sources produce identical metering: one +1, one -1, same ids + records := emitter.Records() + require.Len(t, records, 2) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + }) + } +} diff --git a/core/services/workflows/syncer/v2/helpers.go b/core/services/workflows/syncer/v2/helpers.go index e8be21d6d7f..cefb2b220fb 100644 --- a/core/services/workflows/syncer/v2/helpers.go +++ b/core/services/workflows/syncer/v2/helpers.go @@ -11,6 +11,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" ) @@ -78,6 +79,14 @@ func (m *testEvtHandler) EmitActivationAbandoned(context.Context, Event, eventsv return nil } +func (m *testEvtHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + return nil, nil +} + +func (m *testEvtHandler) ReleaseOrphanedSpec(context.Context, string, string) error { + return nil +} + func (m *testEvtHandler) ClearEvents() { m.mux.Lock() defer m.mux.Unlock() diff --git a/core/services/workflows/syncer/v2/mocks/orm.go b/core/services/workflows/syncer/v2/mocks/orm.go index a4d02addf29..5a6e40bdbfb 100644 --- a/core/services/workflows/syncer/v2/mocks/orm.go +++ b/core/services/workflows/syncer/v2/mocks/orm.go @@ -23,21 +23,33 @@ func (_m *ORM) EXPECT() *ORM_Expecter { } // DeleteWorkflowSpec provides a mock function with given fields: ctx, id -func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) error { +func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteWorkflowSpec") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + var r0 *job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*job.WorkflowSpec, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *job.WorkflowSpec); ok { r0 = rf(ctx, id) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*job.WorkflowSpec) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // ORM_DeleteWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteWorkflowSpec' @@ -59,12 +71,59 @@ func (_c *ORM_DeleteWorkflowSpec_Call) Run(run func(ctx context.Context, id stri return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 error) *ORM_DeleteWorkflowSpec_Call { +func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 *job.WorkflowSpec, _a1 error) *ORM_DeleteWorkflowSpec_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) (*job.WorkflowSpec, error)) *ORM_DeleteWorkflowSpec_Call { + _c.Call.Return(run) + return _c +} + +// PauseWorkflowSpec provides a mock function with given fields: ctx, id +func (_m *ORM) PauseWorkflowSpec(ctx context.Context, id string) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for PauseWorkflowSpec") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ORM_PauseWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PauseWorkflowSpec' +type ORM_PauseWorkflowSpec_Call struct { + *mock.Call +} + +// PauseWorkflowSpec is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *ORM_Expecter) PauseWorkflowSpec(ctx interface{}, id interface{}) *ORM_PauseWorkflowSpec_Call { + return &ORM_PauseWorkflowSpec_Call{Call: _e.mock.On("PauseWorkflowSpec", ctx, id)} +} + +func (_c *ORM_PauseWorkflowSpec_Call) Run(run func(ctx context.Context, id string)) *ORM_PauseWorkflowSpec_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) Return(_a0 error) *ORM_PauseWorkflowSpec_Call { _c.Call.Return(_a0) return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_DeleteWorkflowSpec_Call { +func (_c *ORM_PauseWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_PauseWorkflowSpec_Call { _c.Call.Return(run) return _c } @@ -175,6 +234,64 @@ func (_c *ORM_GetWorkflowSpec_Call) RunAndReturn(run func(context.Context, strin return _c } +// GetWorkflowSpecList provides a mock function with given fields: ctx +func (_m *ORM) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetWorkflowSpecList") + } + + var r0 []*job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*job.WorkflowSpec, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*job.WorkflowSpec); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*job.WorkflowSpec) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ORM_GetWorkflowSpecList_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetWorkflowSpecList' +type ORM_GetWorkflowSpecList_Call struct { + *mock.Call +} + +// GetWorkflowSpecList is a helper method to define mock.On call +// - ctx context.Context +func (_e *ORM_Expecter) GetWorkflowSpecList(ctx interface{}) *ORM_GetWorkflowSpecList_Call { + return &ORM_GetWorkflowSpecList_Call{Call: _e.mock.On("GetWorkflowSpecList", ctx)} +} + +func (_c *ORM_GetWorkflowSpecList_Call) Run(run func(ctx context.Context)) *ORM_GetWorkflowSpecList_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ORM_GetWorkflowSpecList_Call) Return(_a0 []*job.WorkflowSpec, _a1 error) *ORM_GetWorkflowSpecList_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ORM_GetWorkflowSpecList_Call) RunAndReturn(run func(context.Context) ([]*job.WorkflowSpec, error)) *ORM_GetWorkflowSpecList_Call { + _c.Call.Return(run) + return _c +} + // UpsertWorkflowSpec provides a mock function with given fields: ctx, spec func (_m *ORM) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { ret := _m.Called(ctx, spec) diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index ba24c73a603..ffb5af4c8cc 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -28,8 +28,10 @@ import ( "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" "github.com/smartcontractkit/chainlink-evm/pkg/config" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/versioning" + wftypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" ) const name = "WorkflowRegistrySyncer" @@ -142,6 +144,8 @@ type evtHandler interface { Handle(ctx context.Context, event Event) error EmitActivationAbandoned(ctx context.Context, event Event, reason eventsv2.ActivationAbandonReason, activationErr error, retryCount int32) error + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) + ReleaseOrphanedSpec(ctx context.Context, workflowID, owner string) error } type donNotifier interface { @@ -839,6 +843,14 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) // Process each source independently to isolate failures totalWorkflowsFetched := 0 reconcileReport := newReconcileReport() + // allMetadataIDs is the union of every source's (post-shard) + // workflow IDs this tick; it is the source of truth for the + // downtime-orphan sweep. sourceFetchFailed stays false only when + // every source fetched successfully, so the sweep never runs on + // incomplete metadata (it would risk deleting specs whose workflow + // belongs to a source that failed to fetch). + allMetadataIDs := make(map[string]struct{}) + sourceFetchFailed := false for _, source := range w.workflowSources { sourceName := source.Name() @@ -859,6 +871,7 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) w.metrics.recordSourceFetch(ctx, sourceName, len(workflows), duration, fetchErr) if fetchErr != nil { + sourceFetchFailed = true w.lggr.Errorw("Failed to fetch from source, skipping reconciliation for this source", "source", sourceName, "error", fetchErr, "durationMs", duration.Milliseconds()) // KEY: Skip this source entirely - no events generated, no deletions @@ -875,6 +888,7 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) if w.shardingEnabled { filteredWorkflowsMetadata, err = w.filterWorkflowsByShard(ctx, workflows) if err != nil { + sourceFetchFailed = true w.lggr.Errorw("failed to filter workflows by shard", "err", err, "source", sourceName) @@ -888,6 +902,10 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + for _, wfMeta := range filteredWorkflowsMetadata { + allMetadataIDs[wfMeta.WorkflowID.Hex()] = struct{}{} + } + // Generate events only for this source's engines (using sourceIdentifier for engine registry lookups) events, genErr := w.generateReconciliationEvents(ctx, pendingEvents, filteredWorkflowsMetadata, head, sourceIdentifier) if genErr != nil { @@ -998,6 +1016,10 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + if !sourceFetchFailed && len(w.workflowSources) > 0 { + w.reconcileOrphanedSpecs(ctx, allMetadataIDs) + } + w.metrics.recordFetchedWorkflows(ctx, totalWorkflowsFetched) w.lggr.Debugw("reconciled events", "report", reconcileReport) @@ -1019,6 +1041,44 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) } } +// reconcileOrphanedSpecs releases persisted specs whose workflowID is +// absent from all sources. This catches workflows deleted while this +// node is down or restarting. Released through ReleaseOrphanedSpec, they take the +// existing delete path (artifact cleanup and -1 metering delta) but must +// not emit workflow status changed events. +func (w *workflowRegistry) reconcileOrphanedSpecs(ctx context.Context, liveWorkflowIDs map[string]struct{}) { + specs, err := w.handler.GetWorkflowSpecList(ctx) + if err != nil { + w.lggr.Warnw("failed to list persisted workflow specs for orphan reconciliation; skipping sweep", "err", err) + return + } + for _, spec := range specs { + if spec == nil || spec.WorkflowID == "" { + continue + } + if _, live := liveWorkflowIDs[spec.WorkflowID]; live { + continue + } + wfIDBytes, derr := hex.DecodeString(spec.WorkflowID) + if derr != nil || len(wfIDBytes) != len(wftypes.WorkflowID{}) { + w.lggr.Warnw("orphan sweep: skipping unparseable persisted workflow_id", "workflowID", spec.WorkflowID, "err", derr) + continue + } + var wfID wftypes.WorkflowID + copy(wfID[:], wfIDBytes) + if _, engineFound := w.engineRegistry.Get(wfID); engineFound { + // Engine-owned: the reconciliation path deletes it with drain/retry + // machinery. The sweep only handles engine-less leftovers. + continue + } + w.lggr.Debugw("orphaned workflow spec: persisted locally but absent from registry metadata; releasing", + "workflowID", spec.WorkflowID, "owner", spec.WorkflowOwner) + if rerr := w.handler.ReleaseOrphanedSpec(ctx, spec.WorkflowID, spec.WorkflowOwner); rerr != nil { + w.lggr.Warnw("failed to release orphaned workflow spec", "workflowID", spec.WorkflowID, "err", rerr) + } + } +} + // getTicker returns the ticker that the workflowRegistry will use to poll for events. If the ticker // is nil, then a default ticker is returned. func (w *workflowRegistry) getTicker(d time.Duration) <-chan time.Time { diff --git a/core/services/workflows/syncer/v2/workflow_registry_test.go b/core/services/workflows/syncer/v2/workflow_registry_test.go index b1a2fd02575..9317b3c1a57 100644 --- a/core/services/workflows/syncer/v2/workflow_registry_test.go +++ b/core/services/workflows/syncer/v2/workflow_registry_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" commonCap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -20,8 +21,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" + eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" wfTypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" @@ -1997,3 +2000,99 @@ func TestWorkflowRegistry_getTicker_WithTickerOverride(t *testing.T) { tickerCh := wr.getTicker(10 * time.Second) require.Equal(t, (<-chan time.Time)(customCh), tickerCh) } + +type orphanRelease struct { + workflowID string + owner string +} + +// orphanSweepFakeHandler is a minimal evtHandler that returns a fixed spec list +// and records every Handle / ReleaseOrphanedSpec call, so reconcileOrphanedSpecs +// can be unit-tested in isolation. +type orphanSweepFakeHandler struct { + mu sync.Mutex + specs []*job.WorkflowSpec + handled []Event + released []orphanRelease +} + +func (h *orphanSweepFakeHandler) Close() error { return nil } +func (h *orphanSweepFakeHandler) Start(context.Context) error { return nil } +func (h *orphanSweepFakeHandler) Handle(_ context.Context, event Event) error { + h.mu.Lock() + defer h.mu.Unlock() + h.handled = append(h.handled, event) + return nil +} +func (h *orphanSweepFakeHandler) EmitActivationAbandoned(context.Context, Event, eventsv2.ActivationAbandonReason, error, int32) error { + return nil +} +func (h *orphanSweepFakeHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + return h.specs, nil +} +func (h *orphanSweepFakeHandler) ReleaseOrphanedSpec(_ context.Context, workflowID, owner string) error { + h.mu.Lock() + defer h.mu.Unlock() + h.released = append(h.released, orphanRelease{workflowID: workflowID, owner: owner}) + return nil +} +func (h *orphanSweepFakeHandler) Handled() []Event { + h.mu.Lock() + defer h.mu.Unlock() + cp := make([]Event, len(h.handled)) + copy(cp, h.handled) + return cp +} +func (h *orphanSweepFakeHandler) Released() []orphanRelease { + h.mu.Lock() + defer h.mu.Unlock() + cp := make([]orphanRelease, len(h.released)) + copy(cp, h.released) + return cp +} + +// a persisted spec whose workflowID is absent from the registry metadata +// (the union of all sources) and has no running engine is a downtime orphan — +// it is released through ReleaseOrphanedSpec. Specs still present in metadata +// (including paused workflows, which remain in metadata) are left alone. +func Test_reconcileOrphanedSpecs_ReleasesEnginelessOrphansOnly(t *testing.T) { + t.Parallel() + liveID := wfTypes.WorkflowID{1} + orphanID := wfTypes.WorkflowID{2} + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, + }} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: NewEngineRegistry()} + + // liveWorkflowIDs is the union of all sources' metadata this tick; the + // live workflow (present in metadata) is retained, the orphan is released. + live := map[string]struct{}{liveID.Hex(): {}} + w.reconcileOrphanedSpecs(t.Context(), live) + + assert.Empty(t, h.Handled(), "orphan sweep must not dispatch events through Handle") + released := h.Released() + require.Len(t, released, 1, "exactly the orphan is released") + assert.Equal(t, orphanID.Hex(), released[0].workflowID) + assert.Equal(t, "aabbccdd", released[0].owner) +} + +// An orphan with a registered engine is skipped: the engine-based deletion +// path handles it (with drain/retry machinery). +func Test_reconcileOrphanedSpecs_SkipsOrphansWithEngines(t *testing.T) { + t.Parallel() + orphanID := wfTypes.WorkflowID{3} + er := NewEngineRegistry() + require.NoError(t, er.Add(orphanID, "test-source", &mockService{})) + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, + }} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: er} + + // orphanID is absent from metadata (orphan), but has a running engine. + live := map[string]struct{}{} + w.reconcileOrphanedSpecs(t.Context(), live) + + assert.Empty(t, h.Handled(), "no events dispatched") + assert.Empty(t, h.Released(), "engine-owned orphan must not be released by the sweep") +} diff --git a/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql new file mode 100644 index 00000000000..9758ac3d9e6 --- /dev/null +++ b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE workflow_specs_v2 ADD COLUMN registered_at bigint NOT NULL DEFAULT 0; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN registered_at; diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index f0e139720dd..cc77a2a4f6d 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 6bd020acb91..3f09ea51413 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -395,6 +395,16 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +NumericTenantID = '42' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'clp-cre-wf-zone-a-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index c5ab5449c14..b7eb3ab1365 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/deployment/go.mod b/deployment/go.mod index 75c28564d9c..db658bafe25 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -451,7 +451,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/deployment/go.sum b/deployment/go.sum index bd8a2a4dd73..317618ad69f 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1460,8 +1460,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 177c10a107f..065170097be 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2660,6 +2660,71 @@ Prefixes = ["go_"] # Default Prefixes is a set of filters to restrict which prometheus metrics are forwarded based on prefix matching. By default, we only forward the go runtime metrics. Empty means forward everything. +## Metering +```toml +[Metering] +MeterRecordsEnabled = false # Default +MeterSnapshotsEnabled = false # Default +Product = 'cre' # Default +Tenant = '' # Default +NumericTenantID = '' # Default +Environment = '' # Default +Zone = '' # Default +NodeID = '' # Default +``` +Metering configures durable resource metering emission and the coarse +deployment/node identity dimensions stamped on emitted MeterRecords and +MeterSnapshots. + +### MeterRecordsEnabled +```toml +MeterRecordsEnabled = false # Default +``` +MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + +### MeterSnapshotsEnabled +```toml +MeterSnapshotsEnabled = false # Default +``` +MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +Requires MeterRecordsEnabled = true. + +### Product +```toml +Product = 'cre' # Default +``` +Product is the deployment product identity dimension, e.g. 'cre'. + +### Tenant +```toml +Tenant = '' # Default +``` +Tenant is the human-readable tenant name, e.g. 'mainline'. + +### NumericTenantID +```toml +NumericTenantID = '' # Default +``` +NumericTenantID is the numbered tenant identifier represented as a string. + +### Environment +```toml +Environment = '' # Default +``` +Environment is the deployment environment identity dimension, e.g. 'production'. + +### Zone +```toml +Zone = '' # Default +``` +Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. + +### NodeID +```toml +NodeID = '' # Default +``` +NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). + ## CRE.Streams ```toml [CRE.Streams] diff --git a/go.mod b/go.mod index a52be7d8465..23a42bebf27 100644 --- a/go.mod +++ b/go.mod @@ -100,6 +100,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260714170805-29c5577b5f55 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd @@ -355,7 +356,6 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20260408092456-3c6369888d4a // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect diff --git a/go.sum b/go.sum index 45707b0860d..0803a983b1e 100644 --- a/go.sum +++ b/go.sum @@ -1201,8 +1201,8 @@ github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546- github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 0745d0a8af1..d752d08049d 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -62,7 +62,7 @@ require ( require ( github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect ) require ( diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 44debad902e..a0fef075865 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1447,8 +1447,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 568305d675b..23e19adebf4 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -39,6 +39,13 @@ require ( golang.org/x/sync v0.22.0 ) +require ( + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect + github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect + github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect +) + require ( cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -125,7 +132,6 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -503,11 +509,8 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect - github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect - github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 // indirect github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260716141634-c0cc05ed05d8 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 558a1a8aef1..3961f75de92 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1683,8 +1683,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index a20b064688e..ecdb963d97c 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -40,6 +40,7 @@ type LoopRegistry struct { autoPPROF config.AutoPprof cfgTracing config.Tracing cfgTelemetry config.Telemetry + cfgMetering config.Metering telemetryAuthHeaders map[string]string telemetryAuthPubKeyHex string cfgLOOPP config.LOOPP @@ -47,7 +48,7 @@ type LoopRegistry struct { func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, dbConfig config.Database, mercury de.Mercury, pyroscope config.Pyroscope, autoPPROF config.AutoPprof, tracing config.Tracing, telemetry config.Telemetry, - telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { + metering config.Metering, telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { return &LoopRegistry{ registry: map[string]*RegisteredLoop{}, lggr: logger.Named(lggr, "LoopRegistry"), @@ -59,6 +60,7 @@ func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, db autoPPROF: autoPPROF, cfgTracing: tracing, cfgTelemetry: telemetry, + cfgMetering: metering, telemetryAuthHeaders: telemetryAuthHeaders, telemetryAuthPubKeyHex: telemetryAuthPubKeyHex, cfgLOOPP: looppCfg, @@ -176,6 +178,20 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { envCfg.TelemetryPrometheusBridgeEnabled = m.cfgTelemetry.PrometheusBridge().Enabled() envCfg.TelemetryPrometheusBridgePrefixes = m.cfgTelemetry.PrometheusBridge().Prefixes() } + + // Metering config (emission toggles + deployment/node identity dimensions) + // is passed over the env channel to every LOOP plugin, rather than the + // standard-capabilities boundary. + if m.cfgMetering != nil { + envCfg.MeterRecordsEnabled = m.cfgMetering.MeterRecordsEnabled() + envCfg.MeterSnapshotsEnabled = m.cfgMetering.MeterSnapshotsEnabled() + envCfg.MeterProduct = m.cfgMetering.Product() + envCfg.MeterTenant = m.cfgMetering.Tenant() + envCfg.MeterNumericTenantID = m.cfgMetering.NumericTenantID() + envCfg.MeterEnvironment = m.cfgMetering.Environment() + envCfg.MeterZone = m.cfgMetering.Zone() + envCfg.MeterNodeID = m.cfgMetering.NodeID() + } m.lggr.Debugf("Registered loopp %q with port %d", id, envCfg.PrometheusPort) // Add auth header after logging config diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index 28d41bf129a..490f68ae218 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -121,6 +121,17 @@ func (m mockCfgTelemetry) PrometheusBridge() config.PrometheusBridge { return mockPrometheusBridge{} } +type mockCfgMetering struct{} + +func (m mockCfgMetering) MeterRecordsEnabled() bool { return true } +func (m mockCfgMetering) MeterSnapshotsEnabled() bool { return true } +func (m mockCfgMetering) Product() string { return "cre" } +func (m mockCfgMetering) Tenant() string { return "mainline" } +func (m mockCfgMetering) NumericTenantID() string { return "42" } +func (m mockCfgMetering) Environment() string { return "production" } +func (m mockCfgMetering) Zone() string { return "wf-zone-a" } +func (m mockCfgMetering) NodeID() string { return "clp-cre-wf-zone-a-1" } + type mockPrometheusBridge struct{} func (m mockPrometheusBridge) Enabled() bool { return true } @@ -214,6 +225,7 @@ func TestLoopRegistry_Register(t *testing.T) { mockCfgMercury := &mockCfgMercury{} mockCfgTracing := &mockCfgTracing{} mockCfgTelemetry := &mockCfgTelemetry{} + mockCfgMetering := &mockCfgMetering{} registry := make(map[string]*RegisteredLoop) // Create a LoopRegistry instance with mockCfgTracing @@ -226,6 +238,7 @@ func TestLoopRegistry_Register(t *testing.T) { cfgMercury: mockCfgMercury, cfgTracing: mockCfgTracing, cfgTelemetry: mockCfgTelemetry, + cfgMetering: mockCfgMetering, } // Test case 1: Register new loop @@ -279,6 +292,14 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, 512, envCfg.TelemetryLogExportMaxBatchSize) require.Equal(t, 5*time.Second, envCfg.TelemetryLogExportInterval) require.Equal(t, 2048, envCfg.TelemetryLogMaxQueueSize) + require.True(t, envCfg.MeterRecordsEnabled) + require.True(t, envCfg.MeterSnapshotsEnabled) + require.Equal(t, "cre", envCfg.MeterProduct) + require.Equal(t, "mainline", envCfg.MeterTenant) + require.Equal(t, "42", envCfg.MeterNumericTenantID) + require.Equal(t, "production", envCfg.MeterEnvironment) + require.Equal(t, "wf-zone-a", envCfg.MeterZone) + require.Equal(t, "clp-cre-wf-zone-a-1", envCfg.MeterNodeID) require.Equal(t, []string{"event_id"}, envCfg.TelemetryMetricViewsDenyAttributes) require.NotNil(t, envCfg.TelemetryMetricCardinalityLimit) require.Equal(t, 100000, *envCfg.TelemetryMetricCardinalityLimit) diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 4a36fc9f5c3..03fe462e86f 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -474,7 +474,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 3ceacee28f7..148d2d980ab 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1597,8 +1597,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index e690f4f2393..a2b8bff6cd6 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -247,7 +247,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260720132736-e99278bfdc96 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 25cc0d85d70..9f3b24b4fef 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1802,8 +1802,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 821f2f5a76d..0b79050ab19 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -524,6 +524,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index 534841af1bc..21ba967ff09 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -389,6 +389,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index ce093f64219..0ddff376935 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -450,6 +450,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 1296b4e7d72..1011964d249 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 74b11162020..8ad9fe716ed 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 059d6ab65bd..1231e06b1a6 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 865b4aa0f56..51da7740280 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -535,6 +535,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index b09445dc279..7739dc51fd6 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -418,6 +418,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 1f9c08f31f0..be924e1de39 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -429,6 +429,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index e884c6f7efe..8ef8d7be402 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -430,6 +430,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 07c178ad53b..09e11128a1c 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -412,6 +412,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200