From 7bafc6bcff39764d6d62e83d9b250f964470fef9 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 15 Jun 2026 14:53:53 -0400 Subject: [PATCH 01/14] adding resource manager for wf size --- core/config/env/env.go | 4 + core/services/chainlink/application.go | 10 + .../cre/confidential_relay_peerid_test.go | 1 + core/services/cre/cre.go | 83 ++++ .../services/standardcapabilities/delegate.go | 26 + .../standard_capabilities.go | 55 ++- .../standard_capabilities_test.go | 62 +++ .../workflows/syncer/v2/engine_registry.go | 2 +- core/services/workflows/syncer/v2/handler.go | 240 ++++++++- .../syncer/v2/handler_metering_test.go | 462 ++++++++++++++++++ .../workflows/syncer/v2/handler_test.go | 36 +- go.mod | 7 + plugins/plugins.private.yaml | 38 ++ 13 files changed, 987 insertions(+), 39 deletions(-) create mode 100644 core/services/workflows/syncer/v2/handler_metering_test.go diff --git a/core/config/env/env.go b/core/config/env/env.go index dc5b4951fbc..eed36c80152 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -14,6 +14,10 @@ var ( DatabaseAllowSimplePasswords = Var("CL_DATABASE_ALLOW_SIMPLE_PASSWORDS") IgnorePrereleaseVersionCheck = Var("CL_IGNORE_PRE_RELEASE_VERSION_CHECK") SkipAppVersionCheck = Var("CL_SKIP_APP_VERSION_CHECK") + // MeterRecordsEnabled gates emission of metering.v1.MeterRecord events for + // durable CRE resources. Accepts strconv.ParseBool values; default false. + // Temporary deploy gate; promotion to TOML config is tracked by SHARED-2718. + MeterRecordsEnabled = Var("CL_METER_RECORDS_ENABLED") DatabaseURL = Secret("CL_DATABASE_URL") DatabaseBackupURL = Secret("CL_DATABASE_BACKUP_URL") diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 74168bdd70e..a5ab27d7aed 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -676,6 +676,16 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err atomicSettings, creServices.OCRConfigService, cfg.Capabilities().Local(), + // Host-injected deployment/node metering identity for spawned capability + // LOOPs. Sourced once here from node config: product constant, env/zone + // from [Telemetry.ResourceAttributes], node_id = the same CSA pubkey hex + // the node uses for beholder auth and the engine's node_id. + standardcapabilities.NodeIdentity{ + Product: "cre", + Environment: cfg.Telemetry().ResourceAttributes()["env"], + Zone: cfg.Telemetry().ResourceAttributes()["zone"], + NodeID: csaPubKeyHex, + }, ) delegates[job.StandardCapabilities] = stdcapDelegate if creServices.SetDelegatesDeps != nil { diff --git a/core/services/cre/confidential_relay_peerid_test.go b/core/services/cre/confidential_relay_peerid_test.go index 8b92cfeaeec..77f376a5d74 100644 --- a/core/services/cre/confidential_relay_peerid_test.go +++ b/core/services/cre/confidential_relay_peerid_test.go @@ -51,6 +51,7 @@ func (s stubConfig) Workflows() config.Workflows { return nil } func (s stubConfig) CRE() config.CRE { return nil } func (s stubConfig) P2P() config.P2P { return s.p2p } func (s stubConfig) Sharding() config.Sharding { return nil } +func (s stubConfig) Telemetry() config.Telemetry { return nil } func peerIDFromByte(b byte) p2pkey.PeerID { var id p2pkey.PeerID diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 8a00ee1af6a..c57a247cd9f 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -18,11 +18,13 @@ import ( "github.com/smartcontractkit/chainlink-common/keystore/corekeys/p2pkey" "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/billing" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" nodeauthjwt "github.com/smartcontractkit/chainlink-common/pkg/nodeauth/jwt" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -43,6 +45,7 @@ import ( remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" capStreams "github.com/smartcontractkit/chainlink/v2/core/capabilities/streams" "github.com/smartcontractkit/chainlink/v2/core/config" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/ocr/capregconfig" @@ -297,6 +300,10 @@ func (s *Services) newSubservices( return srvs, nil } + // Build the syncer's base metering identity once from node config + CSA key; + // the handler resolves the workflow DON id later from the don notifier. + meterIdentity := newSyncerMeterIdentity(cfg, keyStore, lggr) + wfSyncer, billingClient, wfSyncerSrvcs, err := newWorkflowRegistrySyncer( cfg, relayerChainInterops, @@ -310,6 +317,7 @@ func (s *Services) newSubservices( opts.LimitsFactory, s.OrgResolver, s.GatewayConnectorWrapper, + meterIdentity, ) if err != nil { return nil, err @@ -338,6 +346,7 @@ type Config interface { CRE() config.CRE P2P() config.P2P Sharding() config.Sharding + Telemetry() config.Telemetry } // RelayerChainInterops is the minimal interface needed for relayer chain interops @@ -718,6 +727,67 @@ func newBillingClient(lggr logger.Logger, cfg Config, opts Opts) (metering.Billi return billing.NewWorkflowClient(lggr, cfg.Billing().URL(), workflowOpts...) } +// Metering identity sourcing constants. meterProduct is the deployment product +// dimension for all CRE metering records. The environment/zone dimensions are +// read from the node's [Telemetry.ResourceAttributes] map under these keys, the +// same map the host injects into trigger LOOPs (see core/services/ +// standardcapabilities), so a node's engine and its trigger LOOPs agree on the +// coarse identity. +const ( + meterProduct = "cre" + resourceAttrEnvKey = "env" + resourceAttrZoneKey = "zone" +) + +// nodeCSAPublicKeyHex returns the node's CSA public key as hex — the canonical +// node_id used uniformly across a node's engine and its trigger LOOPs (matching +// keystore.BuildBeholderAuth, which derives the beholder auth pubkey from the +// same default CSA key). A node has at most one CSA key; an empty string is +// returned (with a warning) when none is available, so metering degrades to an +// empty node_id rather than failing node startup. +func nodeCSAPublicKeyHex(keyStore Keystore, lggr logger.Logger) string { + keys, err := keyStore.CSA().GetAll() + if err != nil || len(keys) == 0 { + lggr.Warnw("no CSA key available for metering node_id; node_id will be empty", "err", err) + return "" + } + return keys[0].PublicKeyString() +} + +// newSyncerMeterIdentity builds the syncer's base metering identity from node +// config: product is the constant, environment/zone come from +// [Telemetry.ResourceAttributes], and node_id is the CSA public key. The +// workflow DON id (don_id) is resolved later by the handler from the don +// notifier (the engine runs on the workflow DON), so it is intentionally left +// empty here. Service/Resource/ResourceType are stamped by WithIdentity. +func newSyncerMeterIdentity(cfg Config, keyStore Keystore, lggr logger.Logger) resourcemanager.ResourceIdentity { + attrs := cfg.Telemetry().ResourceAttributes() + return resourcemanager.ResourceIdentity{ + Product: meterProduct, + Environment: attrs[resourceAttrEnvKey], + Zone: attrs[resourceAttrZoneKey], + NodeID: nodeCSAPublicKeyHex(keyStore, lggr), + } +} + +// meterRecordsEnabled reads the CL_METER_RECORDS_ENABLED env var gating emission of +// metering.v1.MeterRecord events. Unset, empty, or invalid values disable emission. +// Promotion of this gate to TOML config is tracked by SHARED-2718. +func meterRecordsEnabled(lggr logger.Logger) bool { + v := env.MeterRecordsEnabled.Get() + if v == "" { + return false + } + enabled, err := strconv.ParseBool(v) + if err != nil { + // Warn, not error, matching the capability producers: a bad gate value + // disables emission but must never disturb the node. + lggr.Warnw("Invalid CL_METER_RECORDS_ENABLED value; meter record emission disabled", "value", v, "err", err) + return false + } + return enabled +} + func newShardOrchestratorClient(cfg Config, lggr logger.Logger) (*shardorchestrator.Client, error) { shardID := cfg.Sharding().ShardIndex() if shardID == 0 { @@ -933,6 +1003,7 @@ func newWorkflowRegistrySyncerV2( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, []commonsrv.Service, error) { capCfg := cfg.Capabilities() wfReg := capCfg.WorkflowRegistry() @@ -1017,6 +1088,16 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithLocalSecretOverrides(lggr, cfg.CRE().LocalSecretOverrides()), syncerV2.WithShardExecutionGuard(shardOrchestratorClient, shardingEnabled, shardIndex), syncerV2.WithShardRoutingSteady(shardRoutingSteady), + // The handler owns this ResourceManager's lifecycle (starts it, registers + // itself as the snapshotted Meterable, closes it). A positive + // SnapshotInterval enables the periodic absolute-state snapshot loop; the + // RM is otherwise a no-op when metering is disabled. + syncerV2.WithResourceManager(resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + Enabled: meterRecordsEnabled(lggr), + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + })), + syncerV2.WithIdentity(meterIdentity), } mc := capCfg.WorkflowRegistry().ModuleCache() @@ -1161,6 +1242,7 @@ func newWorkflowRegistrySyncer( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, metering.BillingClient, []commonsrv.Service, error) { capCfg := cfg.Capabilities() @@ -1209,6 +1291,7 @@ func newWorkflowRegistrySyncer( lf, orgResolver, gatewayConnectorWrapper, + meterIdentity, ) return syncer, billingClient, srvcs, err default: diff --git a/core/services/standardcapabilities/delegate.go b/core/services/standardcapabilities/delegate.go index 63a37c1bcef..9cd5e7275be 100644 --- a/core/services/standardcapabilities/delegate.go +++ b/core/services/standardcapabilities/delegate.go @@ -46,6 +46,23 @@ type RelayGetter interface { GetIDToRelayerMap() map[types.RelayID]loop.Relayer } +// NodeIdentity is the host-injected deployment/node metering identity that the +// Delegate stamps onto every spawned capability's StandardCapabilitiesDependencies, +// mirroring how the engine sources the same dimensions from node config (see +// core/services/cre). It is sourced once at node startup (where the CSA key and +// telemetry config are both available) so operators configure it in one place and +// a node's engine and its trigger LOOPs agree on product/environment/zone/node_id. +type NodeIdentity struct { + // Product is the deployment product, e.g. "cre". + Product string + // Environment is the deployment environment, from [Telemetry.ResourceAttributes]["env"]. + Environment string + // Zone is the deployment zone, from [Telemetry.ResourceAttributes]["zone"]. + Zone string + // NodeID is the node's CSA public key (hex), matching the engine's node_id. + NodeID string +} + type Delegate struct { logger logger.Logger ds sqlutil.DataSource @@ -66,6 +83,7 @@ type Delegate struct { creSettings core.SettingsBroadcaster ocrConfigService capregconfig.OCRConfigService localCfg coreconfig.LocalCapabilities + nodeIdentity NodeIdentity initErr error isNewlyCreatedJob bool @@ -98,6 +116,7 @@ func NewDelegate( creSettings core.SettingsBroadcaster, ocrConfigService capregconfig.OCRConfigService, localCfg coreconfig.LocalCapabilities, + nodeIdentity NodeIdentity, opts ...func(*gateway.RoundRobinSelector), ) *Delegate { initErr := registerOptionalMockStreamsTrigger(logger, localCfg, registry) @@ -125,6 +144,7 @@ func NewDelegate( creSettings: creSettings, ocrConfigService: ocrConfigService, localCfg: localCfg, + nodeIdentity: nodeIdentity, initErr: initErr, selectorOpts: opts, } @@ -408,6 +428,12 @@ func (d *Delegate) NewServices( CRESettings: d.creSettings, TriggerEventStore: triggercap.NewTriggerEventStore(d.ds), CapabilityDonID: capabilityDonID, + // Host-injected deployment/node metering identity, delivered to trigger + // LOOPs through the standardized Initialise channel. + Product: d.nodeIdentity.Product, + Environment: d.nodeIdentity.Environment, + Zone: d.nodeIdentity.Zone, + NodeID: d.nodeIdentity.NodeID, } standardCapability := NewStandardCapabilities(log, command, configJSON, d.cfg, dependencies) diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index 3d7e7f8032c..bc7f600e24c 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -46,6 +46,13 @@ type StandardCapabilities struct { // at Initialise time. Zero means the host did not resolve one; the plugin // will fall back to capability-registry lookup. capabilityDonID uint32 + // Host-injected deployment/node metering identity, captured from the deps + // passed at construction and re-delivered to the LOOP through the deps built + // for Initialise below. + product string + environment string + zone string + nodeID string capabilitiesLoop *loop.StandardCapabilitiesService @@ -77,17 +84,51 @@ func NewStandardCapabilities( creSettings: dependencies.CRESettings, triggerEventStore: dependencies.TriggerEventStore, capabilityDonID: dependencies.CapabilityDonID, + product: dependencies.Product, + environment: dependencies.Environment, + zone: dependencies.Zone, + nodeID: dependencies.NodeID, stopChan: make(chan struct{}), readyChan: make(chan struct{}), } } +// initialiseDependencies builds the StandardCapabilitiesDependencies delivered to +// the capability LOOP via Initialise. It re-emits the host-injected metering +// identity (product/environment/zone/node_id) captured at construction so trigger +// LOOPs receive it through the standardized Initialise channel. +func (s *StandardCapabilities) initialiseDependencies() core.StandardCapabilitiesDependencies { + return core.StandardCapabilitiesDependencies{ + Config: s.config, + Store: s.store, + CapabilityRegistry: s.CapabilitiesRegistry, + RelayerSet: s.relayerSet, + OracleFactory: s.oracleFactory, + GatewayConnector: s.gatewayConnector, + P2PKeystore: s.keystore, + OrgResolver: s.orgResolver, + CRESettings: s.creSettings, + TriggerEventStore: s.triggerEventStore, + CapabilityDonID: s.capabilityDonID, + Product: s.product, + Environment: s.environment, + Zone: s.zone, + NodeID: s.nodeID, + } +} + func (s *StandardCapabilities) Start(ctx context.Context) error { return s.StartOnce("StandardCapabilities", func() error { envVars, err := plugins.ParseEnvFile(env.CapabilitiesPlugin.Env.Get()) if err != nil { return fmt.Errorf("failed to parse capabilities env file: %w", err) } + // Pass through the node's meter-record emission gate so capability LOOPPs + // inherit it: plugins.NewCmdFactory builds the child env from CmdConfig.Env + // only and does not inherit os.Environ(). + if v := env.MeterRecordsEnabled.Get(); v != "" { + envVars = append(envVars, fmt.Sprintf("%s=%s", env.MeterRecordsEnabled, v)) + } cmdFn, opts, err := s.pluginRegistrar.RegisterLOOP(plugins.CmdConfig{ ID: s.log.Name(), Cmd: s.command, @@ -119,19 +160,7 @@ func (s *StandardCapabilities) Start(ctx context.Context) error { return } - dependencies := core.StandardCapabilitiesDependencies{ - Config: s.config, - Store: s.store, - CapabilityRegistry: s.CapabilitiesRegistry, - RelayerSet: s.relayerSet, - OracleFactory: s.oracleFactory, - GatewayConnector: s.gatewayConnector, - P2PKeystore: s.keystore, - OrgResolver: s.orgResolver, - CRESettings: s.creSettings, - TriggerEventStore: s.triggerEventStore, - CapabilityDonID: s.capabilityDonID, - } + dependencies := s.initialiseDependencies() if err = s.capabilitiesLoop.Service.Initialise(cctx, dependencies); err != nil { s.log.Errorf("error initialising standard capabilities service: %v", err) return diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index ce0159b54f7..622166a54be 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -74,6 +74,7 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { t.Run("env file unset results in empty CmdConfig.Env", func(t *testing.T) { t.Setenv(string(env.CapabilitiesPlugin.Env), "") + t.Setenv(string(env.MeterRecordsEnabled), "") cfg, err := startAndCapture(t) require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") @@ -83,6 +84,33 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset") }) + t.Run("CL_METER_RECORDS_ENABLED set on the node is passed through to the LOOPP", func(t *testing.T) { + envFile := writeEnvFile(t, "FOO=bar\n") + t.Setenv(string(env.CapabilitiesPlugin.Env), envFile) + t.Setenv(string(env.MeterRecordsEnabled), "true") + + cfg, err := startAndCapture(t) + require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") + require.Contains(t, err.Error(), capturingRegistrarErr) + + require.Contains(t, cfg.Env, "CL_METER_RECORDS_ENABLED=true", + "the node's meter-record emission gate must reach capability LOOPPs, which do not inherit os.Environ()") + require.Contains(t, cfg.Env, "FOO=bar", + "the pass-through must not displace entries from the operator-supplied env file") + }) + + t.Run("CL_METER_RECORDS_ENABLED unset is not forwarded", func(t *testing.T) { + t.Setenv(string(env.CapabilitiesPlugin.Env), "") + t.Setenv(string(env.MeterRecordsEnabled), "") + + cfg, err := startAndCapture(t) + require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") + require.Contains(t, err.Error(), capturingRegistrarErr) + + require.Empty(t, cfg.Env, + "no CL_METER_RECORDS_ENABLED entry should be forwarded when the gate is unset on the node") + }) + t.Run("missing env file fails Start before RegisterLOOP", func(t *testing.T) { missingPath := filepath.Join(t.TempDir(), "does-not-exist.env") t.Setenv(string(env.CapabilitiesPlugin.Env), missingPath) @@ -106,6 +134,40 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { }) } +// TestStandardCapabilities_ForwardsNodeMeteringIdentity asserts that the +// host-injected deployment/node metering identity (Product/Environment/Zone/ +// NodeID) supplied on the deps at construction is re-delivered, unchanged, on the +// StandardCapabilitiesDependencies handed to the capability LOOP at Initialise. +// This is the standardized Initialise channel that gives trigger LOOPs the same +// coarse metering identity the node's engine uses. +func TestStandardCapabilities_ForwardsNodeMeteringIdentity(t *testing.T) { + want := core.StandardCapabilitiesDependencies{ + Product: "cre", + Environment: "staging", + Zone: "wf-zone-a", + NodeID: "0a1b2c3d4e5f", + } + + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + want, + ) + + got := std.initialiseDependencies() + + require.Equal(t, want.Product, got.Product, + "the host-injected product must reach the capability LOOP via Initialise") + require.Equal(t, want.Environment, got.Environment, + "the host-injected environment must reach the capability LOOP via Initialise") + require.Equal(t, want.Zone, got.Zone, + "the host-injected zone must reach the capability LOOP via Initialise") + require.Equal(t, want.NodeID, got.NodeID, + "the host-injected node_id (CSA pubkey) must reach the capability LOOP via Initialise") +} + const capturingRegistrarErr = "capturingRegistrar: stop after capture" // capturingRegistrar records the CmdConfig handed to RegisterLOOP and then diff --git a/core/services/workflows/syncer/v2/engine_registry.go b/core/services/workflows/syncer/v2/engine_registry.go index afd712afe2a..6dd9adfc428 100644 --- a/core/services/workflows/syncer/v2/engine_registry.go +++ b/core/services/workflows/syncer/v2/engine_registry.go @@ -18,7 +18,7 @@ type ServiceWithMetadata struct { services.Service } -// engineEntry holds the engine and its associated source for internal storage +// engineEntry holds the engine and its associated source for internal storage. type engineEntry struct { engine services.Service source string diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 55284f71ed0..6c871f881bc 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "maps" + "strconv" "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel" @@ -19,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -28,6 +31,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/workflows/dontime" generichost "github.com/smartcontractkit/chainlink-common/pkg/workflows/host" "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" @@ -64,6 +68,17 @@ type DrainableService interface { var ErrDrainInProgress = errors.New("drain in progress") +// Service-level metering identity constants for the workflow syncer. Service is +// the stable service constant; Resource/ResourceType identify the workflow_specs_v2 +// pool and its billing unit. The coarse deployment/node/DON dimensions (product, +// environment, zone, don_id, node_id) are supplied at construction via +// WithIdentity; resource_id is set per workflow via WithResourceID. +const ( + meterService = "workflow-syncer-v2" + meterResource = "workflow_specs_v2" + meterResourceType = "operations" +) + // eventHandler is a handler for WorkflowRegistryEvent events. Each event type has a corresponding method that handles the event. type eventHandler struct { services.Service @@ -88,8 +103,25 @@ type eventHandler struct { workflowEncryptionKey workflowkey.Key workflowDonSubscriber capabilities.DonSubscriber billingClient metering.BillingClient - orgResolver orgresolver.OrgResolver - secretsFetcher v2.SecretsFetcher + resourceManager *resourcemanager.ResourceManager + // meterIdentity is the base metering identity for this node's syncer: the + // six coarse dimensions (product, environment, zone, don_id, node_id, + // service) plus the workflow_specs_v2 resource/resource_type. resource_id is + // set per workflow via WithResourceID. It is populated by WithIdentity in + // cre.go from node TOML + CSA + the workflow DON; Service/Resource/ + // ResourceType are forced to the syncer constants regardless of the option. + meterIdentity resourcemanager.ResourceIdentity + // resolvedDonID holds the workflow DON id once resolved from the don notifier + // at start (the engine runs on the workflow DON). It is resolved + // asynchronously so node boot is not blocked while waiting for the DON to be + // set, and read on the hot snapshot/emit paths; an atomic keeps that read + // lock-free. Nil until resolved; baseIdentity folds it into meterIdentity. + resolvedDonID atomic.Pointer[string] + // rmUnregister removes this handler from the ResourceManager's snapshot + // registry; set in start, called in close. Nil until started. + rmUnregister func() + orgResolver orgresolver.OrgResolver + secretsFetcher v2.SecretsFetcher // localSecretOverrides is keyed by owner address; values are secret id -> secret value localSecretOverrides map[string]map[string]string @@ -153,6 +185,30 @@ func WithBillingClient(client metering.BillingClient) func(*eventHandler) { } } +// WithResourceManager overrides the default (disabled) ResourceManager used to +// emit metering.v1.MeterRecord events for the workflow_specs_v2 storage lifecycle. +func WithResourceManager(rm *resourcemanager.ResourceManager) func(*eventHandler) { + return func(e *eventHandler) { + e.resourceManager = rm + } +} + +// WithIdentity supplies the coarse metering identity dimensions sourced once at +// construction in cre.go (product, environment, zone from node TOML; node_id = +// CSA pubkey hex; don_id = the workflow DON the engine runs on). The syncer +// Service/Resource/ResourceType are stable constants and always overwrite +// whatever the caller passes, so the option carries only the deployment/node/DON +// dimensions; resource_id is left empty and set per workflow via WithResourceID. +func WithIdentity(id resourcemanager.ResourceIdentity) func(*eventHandler) { + return func(e *eventHandler) { + id.Service = meterService + id.Resource = meterResource + id.ResourceType = meterResourceType + id.ResourceID = "" + e.meterIdentity = id + } +} + func WithShardExecutionGuard(client shardorchestrator.ClientInterface, shardingEnabled bool, shardID uint32) func(*eventHandler) { return func(e *eventHandler) { e.shardOrchestratorClient = client @@ -306,7 +362,15 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode + resourceManager: resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}), // default to disabled, enable via WithResourceManager + // Default identity carries only the service-level constants; the coarse + // deployment/node/DON dimensions are filled in by WithIdentity in cre.go. + meterIdentity: resourcemanager.ResourceIdentity{ + Service: meterService, + Resource: meterResource, + ResourceType: meterResourceType, + }, + tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode } metricsInst, metricsErr := newMetrics() if metricsErr != nil { @@ -327,20 +391,72 @@ func NewEventHandler( return eh, nil } -func (h *eventHandler) start(_ context.Context) error { +func (h *eventHandler) start(ctx context.Context) error { if h.moduleLRU != nil { h.moduleLRU.Start() } + // The handler is the single owner of its ResourceManager: it starts the RM + // (which owns the snapshot tick) and registers itself as the Meterable that + // is snapshotted. The RM is a no-op when metering is disabled, so this is + // safe regardless of the gate. + if h.resourceManager != nil { + if err := h.resourceManager.Start(ctx); err != nil { + return fmt.Errorf("failed to start resource manager: %w", err) + } + h.rmUnregister = h.resourceManager.Register(h) + h.resolveWorkflowDonID() + } return nil } +// resolveWorkflowDonID asynchronously resolves the workflow DON id (the engine +// runs on the workflow DON) and folds it into the metering identity. It +// subscribes to the don notifier and stores the first DON's id, so node boot is +// never blocked waiting for the DON to be set. Until it resolves, emitted records +// carry an empty don_id (the host-injection fallback semantics); once resolved, +// every subsequent record and snapshot carries it. Resolution happens at most +// once per start. +func (h *eventHandler) resolveWorkflowDonID() { + if h.workflowDonSubscriber == nil || h.resolvedDonID.Load() != nil { + return + } + h.eng.Go(func(ctx context.Context) { + ch, unsubscribe, err := h.workflowDonSubscriber.Subscribe(ctx) + if err != nil { + h.lggr.Warnw("failed to subscribe to workflow DON for metering identity; don_id will be empty", "err", err) + return + } + defer unsubscribe() + select { + case <-ctx.Done(): + return + case don := <-ch: + donID := strconv.FormatUint(uint64(don.ID), 10) + h.resolvedDonID.Store(&donID) + h.lggr.Debugw("resolved workflow DON id for metering identity", "donID", donID) + } + }) +} + func (h *eventHandler) close() error { if h.moduleLRU != nil { h.moduleLRU.Close() } es := h.engineRegistry.PopAll() - cs := make([]io.Closer, 0, len(es)+1) + // Emit a graceful-close RELEASE for each still-running engine before popping + // them, so a clean shutdown pairs every snapshot RESERVE. Best-effort and + // fail-open: metering must never gate shutdown. + h.emitGracefulCloseReleases(context.Background(), es) + // Stop snapshotting this handler, then close the ResourceManager service. + if h.rmUnregister != nil { + h.rmUnregister() + h.rmUnregister = nil + } + cs := make([]io.Closer, 0, len(es)+2) cs = append(cs, h.engineLimiters) + if h.resourceManager != nil { + cs = append(cs, h.resourceManager) + } for _, e := range es { cs = append(cs, e) } @@ -500,7 +616,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { } }() - if herr = h.workflowDeletedEvent(ctx, payload); herr != nil { + if herr = h.workflowDeletedEvent(ctx, payload, WorkflowDeleted); herr != nil { if errors.Is(herr, ErrDrainInProgress) { logCustMsg(ctx, cma, fmt.Sprintf("workflow deletion deferred: %v", herr), h.lggr) } else { @@ -524,7 +640,7 @@ func (h *eventHandler) workflowActivatedEvent( ) error { // Convert WorkflowActivatedEvent to WorkflowRegisteredEvent since they have identical fields registeredPayload := WorkflowRegisteredEvent(payload) - return h.workflowRegisteredEvent(ctx, registeredPayload) + return h.workflowRegisteredEvent(ctx, registeredPayload, WorkflowActivated) } // workflowRegisteredEvent handles the WorkflowRegisteredEvent event type. @@ -532,9 +648,13 @@ func (h *eventHandler) workflowActivatedEvent( // workflowRegisteredEvent proceeds in two phases: // - phase 1 synchronizes the database state // - phase 2 synchronizes the state of the engine registry. +// originatingEvent names the event that triggered this call (e.g. WorkflowActivated +// when delegated from workflowActivatedEvent) and identifies the originating intent +// in emitted meter records. func (h *eventHandler) workflowRegisteredEvent( ctx context.Context, payload WorkflowRegisteredEvent, + originatingEvent WorkflowRegistryEventName, ) error { ctx, span := h.tracer.Start(ctx, "workflow_registered", trace.WithAttributes( @@ -557,6 +677,7 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, originatingEvent, payload.WorkflowID.Hex()) spec = newSpec case spec.WorkflowID != payload.WorkflowID.Hex(): @@ -564,6 +685,7 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, originatingEvent, payload.WorkflowID.Hex()) spec = newSpec case spec.Status != status: @@ -571,6 +693,7 @@ func (h *eventHandler) workflowRegisteredEvent( if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { return fmt.Errorf("failed to update workflow spec: %w", innerErr) } + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, originatingEvent, payload.WorkflowID.Hex()) } // Next, let's synchronize the engine. @@ -820,13 +943,17 @@ func (h *eventHandler) workflowPausedEvent( ctx context.Context, payload WorkflowPausedEvent, ) error { - return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}) + return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}, WorkflowPaused) } // workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. +// originatingEvent names the event that triggered this call (e.g. WorkflowPaused +// when delegated from workflowPausedEvent) and identifies the originating intent +// in emitted meter records. func (h *eventHandler) workflowDeletedEvent( ctx context.Context, payload WorkflowDeletedEvent, + originatingEvent WorkflowRegistryEventName, ) error { // The order in the handler is slightly different to the order in `tryEngineCleanup`. // This is because the engine requires its corresponding DB record to be present to be successfully @@ -865,6 +992,7 @@ func (h *eventHandler) workflowDeletedEvent( if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } + h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, originatingEvent, workflowID) h.cleanupModuleCache(payload.WorkflowID.Hex()) @@ -885,6 +1013,100 @@ func (h *eventHandler) workflowDeletedEvent( return nil } +// emitMeterRecord emits a metering.v1.MeterRecord for a workflow_specs_v2 mutation. +// Callers must invoke it only after the database mutation has succeeded. Emission is +// fail-open: it never returns an error and must never affect event handling. The +// idempotency key is deterministic over workflowID, originatingEvent, and action, so +// a retried event emits a record the billing consumer dedups against the original. +// +// Known lost-record windows (consumer-side reconciliation tracked by SHARED-2141): +// - A node crash between the workflow spec database write and this emit loses the +// record (e.g. the RESERVE for a new spec): the syncer has no recovery path that +// re-emits records for mutations persisted before the crash. +// - A workflow paused or deleted while the node is down loses its RELEASE: +// reconciliation only synthesizes events for engine-registry entries, and the +// restarted node has no engine registered for the already-removed workflow. +func (h *eventHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, originatingEvent WorkflowRegistryEventName, workflowID string) { + if h.resourceManager == nil { + return + } + // resource_id = workflow_id (the syncer has no shared physical resource); the + // originating event name is the event identity that distinguishes lifecycle + // edges in the idempotency key. + identity := h.baseIdentity().WithResourceID(workflowID) + h.resourceManager.EmitMeterRecord(ctx, identity, action, + resourcemanager.NewUtilization(identity, action, 1, string(originatingEvent))) +} + +// baseIdentity returns the handler's metering identity with the workflow DON id +// folded in once it has been resolved from the don notifier. meterIdentity itself +// is immutable after construction (set via WithIdentity); only the DON id is +// learned later, so it is read from an atomic and overlaid here. +func (h *eventHandler) baseIdentity() resourcemanager.ResourceIdentity { + id := h.meterIdentity + if donID := h.resolvedDonID.Load(); donID != nil { + id.DONID = *donID + } + return id +} + +// ResourceIdentity implements resourcemanager.Meterable: it returns the syncer's +// base identity (six coarse dimensions + workflow_specs_v2 resource/resource_type, +// resource_id empty). The ResourceManager uses it as the top-level identity of +// each emitted Snapshot. +func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { + return h.baseIdentity() +} + +// GetUtilization implements resourcemanager.Meterable: it returns one +// SnapshotEntry per running engine, with Value 1 (each running workflow holds one +// workflow_specs_v2 reservation). It is a pure in-memory read over the engine +// registry — each entry's resource_id is the workflow_id, which fully identifies +// the resource (Design 1; reconciling persisted-but-not-running specs is the +// Design 2 follow-up, see snapshotDesign2Followup below). +func (h *eventHandler) GetUtilization(_ context.Context) []resourcemanager.SnapshotEntry { + base := h.baseIdentity() + engines := h.engineRegistry.GetAll() + entries := make([]resourcemanager.SnapshotEntry, 0, len(engines)) + for _, e := range engines { + workflowID := e.WorkflowID.Hex() + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: base.WithResourceID(workflowID), + Value: 1, + }) + } + return entries +} + +// snapshotDesign2Followup documents the deferred reconciliation work. Design 1 +// (implemented here) snapshots only engines that are currently running in this +// node's engine registry. Design 2 — the follow-up — would additionally +// enumerate persisted-but-not-running specs via WorkflowSpecsDS.ListAll so that +// snapshots also account for specs the database has but for which no engine is +// live (e.g. paused specs, or specs whose engine failed to start). That requires +// a store-backed read on the tick (not a cheap in-memory snapshot) and a policy +// for the utilization value of a non-running spec, so it is intentionally left +// out of the in-memory GetUtilization above. Tracked as the syncer Design 2 +// follow-up. +const snapshotDesign2Followup = "Design 2: reconcile persisted-but-not-running specs via WorkflowSpecsDS.ListAll" + +// emitGracefulCloseReleases emits a RELEASE meter record for every engine still +// in the registry at shutdown, so a clean close pairs every snapshot RESERVE. It +// is fail-open like all metering emission. Called from close before the engines +// are popped. +func (h *eventHandler) emitGracefulCloseReleases(ctx context.Context, engines []ServiceWithMetadata) { + if h.resourceManager == nil { + return + } + base := h.baseIdentity() + for _, e := range engines { + workflowID := e.WorkflowID.Hex() + identity := base.WithResourceID(workflowID) + h.resourceManager.EmitMeterRecord(ctx, identity, meteringpb.MeterAction_METER_ACTION_RELEASE, + resourcemanager.NewUtilization(identity, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, string(WorkflowDeleted))) + } +} + // tryEngineCleanup attempts to stop the workflow engine for the given workflow ID. Does nothing if the // workflow engine is not running. func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { @@ -1004,7 +1226,7 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp } } - // Engine is fully initialized, add to registry with source tracking + // Engine is fully initialized, add to registry with source tracking. if err := h.engineRegistry.Add(wid, source, engine); err != nil { if closeErr := engine.Close(); closeErr != nil { return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go new file mode 100644 index 00000000000..d48ed3d90fc --- /dev/null +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -0,0 +1,462 @@ +package v2 + +import ( + "context" + "encoding/hex" + "errors" + "math/big" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + commonlogger "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + + "github.com/smartcontractkit/chainlink/v2/core/capabilities" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" + workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// recordingEmitter is a fake resourcemanager.Emitter that decodes and stores +// every emitted MeterRecord. If err is set, Emit fails instead. +type recordingEmitter struct { + mu sync.Mutex + err error + records []*meteringpb.MeterRecord +} + +func (r *recordingEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.err != nil { + return r.err + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + r.records = append(r.records, &record) + return nil +} + +func (r *recordingEmitter) Records() []*meteringpb.MeterRecord { + r.mu.Lock() + defer r.mu.Unlock() + records := make([]*meteringpb.MeterRecord, len(r.records)) + copy(records, r.records) + return records +} + +func newMeteringResourceManager(t *testing.T, enabled bool, emitter resourcemanager.Emitter) *resourcemanager.ResourceManager { + t.Helper() + return resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + Enabled: enabled, + Emitter: emitter, + }) +} + +// recordingSnapshotEmitter is a fake resourcemanager.Emitter that decodes and +// stores every emitted MeterSnapshot (one per active resource). Used to drive +// the Meterable snapshot path. +type recordingSnapshotEmitter struct { + mu sync.Mutex + snapshots []*meteringpb.MeterSnapshot +} + +func (r *recordingSnapshotEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + r.snapshots = append(r.snapshots, &snapshot) + return nil +} + +func (r *recordingSnapshotEmitter) Snapshots() []*meteringpb.MeterSnapshot { + r.mu.Lock() + defer r.mu.Unlock() + snapshots := make([]*meteringpb.MeterSnapshot, len(r.snapshots)) + copy(snapshots, r.snapshots) + return snapshots +} + +func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithResourceManager(rm), + WithEngineFactoryFn(mockEngineFactory), + ) + require.NoError(t, err) + return h +} + +func requireMeterRecord(t *testing.T, record *meteringpb.MeterRecord, action meteringpb.MeterAction, originatingEvent WorkflowRegistryEventName, workflowID string) { + t.Helper() + require.NotNil(t, record.Identity) + assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) + assert.Equal(t, "workflow_specs_v2", record.Identity.Resource) + assert.Equal(t, "operations", record.Identity.ResourceType) + // resource_id = workflow_id for the syncer (no shared physical resource). + assert.Equal(t, workflowID, record.Identity.ResourceId) + assert.Equal(t, action, record.Action) + assert.NotNil(t, record.Timestamp) + require.NotNil(t, record.Utilization) + assert.Equal(t, int64(1), record.Utilization.Value) + // The idempotency key is derived over the full identity (with resource_id set + // to workflow_id) and the originating event name as the event identity. + wantIdentity := record.Identity + wantID := resourcemanager.ResourceIdentity{ + Product: wantIdentity.Product, + Environment: wantIdentity.Environment, + Zone: wantIdentity.Zone, + DONID: wantIdentity.DonId, + NodeID: wantIdentity.NodeId, + Service: wantIdentity.Service, + Resource: wantIdentity.Resource, + ResourceType: wantIdentity.ResourceType, + ResourceID: wantIdentity.ResourceId, + } + assert.Equal(t, resourcemanager.IdempotencyKey(wantID, action, string(originatingEvent)), record.Utilization.IdempotencyKey) +} + +func Test_meterRecords(t *testing.T) { + t.Parallel() + + wfOwner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + wfOwnerHex := hex.EncodeToString(wfOwner) + + t.Run("registered event creating a new spec emits RESERVE", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{1} + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }, WorkflowRegistered) + require.NoError(t, err) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RESERVE, WorkflowRegistered, wfID.Hex()) + }) + + t.Run("retried registered event emits an identical idempotency key", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + // The stub never returns a stored spec, so each call replays the + // new-spec path exactly as a reprocessed event would. + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{2}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + } + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) + + records := emitter.Records() + require.Len(t, records, 2) + assert.Equal(t, records[0].Utilization.IdempotencyKey, records[1].Utilization.IdempotencyKey) + }) + + t.Run("activated event with existing spec emits UPDATE", func(t *testing.T) { + t.Parallel() + binary := []byte("binary-data") + config := []byte("") + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "wf-name", binary, config, "") + require.NoError(t, err) + wfID := types.WorkflowID(giveWFID) + + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + Workflow: hex.EncodeToString(binary), + Config: string(config), + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusPaused, + WorkflowOwner: wfOwnerHex, + WorkflowName: "wf-name", + }, + }, newMeteringResourceManager(t, true, emitter)) + + err = h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.NoError(t, err) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_UPDATE, WorkflowActivated, wfID.Hex()) + }) + + t.Run("paused event emits RELEASE after artifacts are deleted", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{3} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowPaused, wfID.Hex()) + }) + + t.Run("deleted event emits RELEASE after artifacts are deleted", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{4} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + }) + + t.Run("no record when creating a new spec fails", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{upsertErr: assert.AnError}, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{5}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }, WorkflowRegistered) + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record when a status-only update fails", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{6} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + upsertErr: assert.AnError, + }, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }, WorkflowRegistered) + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record when deleting artifacts fails", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{7} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + deleteErr: assert.AnError, + }, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted) + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record while a delete is deferred by drain; exactly one on the successful retry", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{8} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + }, newMeteringResourceManager(t, true, emitter)) + + drainable := &mockDrainableEngine{} + drainable.activeExecutions.Store(1) + require.NoError(t, h.engineRegistry.Add(wfID, "test-source", drainable)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted) + require.ErrorIs(t, err, ErrDrainInProgress) + assert.Empty(t, emitter.Records()) + + drainable.activeExecutions.Store(0) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + }) + + t.Run("emit failure never fails event handling", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{9} + emitter := &recordingEmitter{err: errors.New("beholder unavailable")} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: wfOwnerHex, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }, WorkflowRegistered)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + assert.Empty(t, emitter.Records()) + }) + + t.Run("disabled resource manager emits nothing", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, false, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{10}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }, WorkflowRegistered)) + assert.Empty(t, emitter.Records()) + }) + + t.Run("snapshot emits one MeterSnapshot per running engine", func(t *testing.T) { + t.Parallel() + emitter := &recordingSnapshotEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + rm := resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, rm) + unregister := rm.Register(h) + t.Cleanup(unregister) + + // Register two engines; the running engine registry is the in-memory source + // the snapshot reads (no per-tick GetWorkflowSpec). + wfID1 := types.WorkflowID{20} + wfID2 := types.WorkflowID{21} + require.NoError(t, h.engineRegistry.Add(wfID1, "test-source", &fakeService{})) + require.NoError(t, h.engineRegistry.Add(wfID2, "test-source", &fakeService{})) + + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.Snapshots()) == 2 + }, time.Second, time.Millisecond) + snapshots := emitter.Snapshots() + require.Len(t, snapshots, 2) + + byWorkflowID := map[string]*meteringpb.MeterSnapshot{} + for _, snap := range snapshots { + require.NotNil(t, snap.Identity) + assert.Equal(t, "workflow-syncer-v2", snap.Identity.Service) + assert.Equal(t, "workflow_specs_v2", snap.Identity.Resource) + require.NotNil(t, snap.Utilization) + assert.Equal(t, int64(1), snap.Utilization.Value) + // resource_id = workflow_id fully identifies the resource; no labels. + byWorkflowID[snap.Identity.ResourceId] = snap + } + require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first running engine") + require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second running engine") + }) + + t.Run("graceful close emits a RELEASE per running engine", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{30} + require.NoError(t, h.engineRegistry.Add(wfID, "test-source", &fakeService{})) + + es := h.engineRegistry.GetAll() + h.emitGracefulCloseReleases(t.Context(), es) + + records := emitter.Records() + require.Len(t, records, 1) + requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + }) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 25d576fe5bb..8dc521088b7 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -247,7 +247,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedURLParameter := "?auth=abc123" defaultValidationFn := func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, _ *mockFetcher) { - err := h.workflowRegisteredEvent(ctx, event) + err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) // Verify the record is updated in the database @@ -389,7 +389,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { validationFn: func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL string, configURL string, ) { - err := h.workflowRegisteredEvent(ctx, event) + err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.Error(t, err) require.ErrorIs(t, err, assert.AnError) }, @@ -427,7 +427,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { me := &mockEngine{} err := h.engineRegistry.Add(wfID, event.Source, me) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event) + err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) }, }, @@ -466,7 +466,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { oldWfIDBytes := [32]byte{0, 1, 2, 3, 5} err := h.engineRegistry.Add(oldWfIDBytes, event.Source, me) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event) + err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) engineInRegistry, ok := h.engineRegistry.Get(wfID) assert.True(t, ok) @@ -505,7 +505,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { validationFn: func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL string, configURL string, ) { - err := h.workflowRegisteredEvent(ctx, event) + err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) // Verify the record is updated in the database @@ -568,7 +568,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { _, err := s.UpsertWorkflowSpec(ctx, entry) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event) + err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) // Verify the record is updated in the database @@ -794,7 +794,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { } ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, event) + err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) assert.Eventually(t, confidential.ran.Load, 10*time.Second, time.Millisecond) @@ -883,7 +883,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { } ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, event) + err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) require.NoError(t, err) assert.Eventually(t, action.ran.Load, 10*time.Second, time.Millisecond) @@ -1109,7 +1109,7 @@ func Test_workflowDeletedHandler(t *testing.T) { ) require.NoError(t, err) ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, active) + err = h.workflowRegisteredEvent(ctx, active, WorkflowRegistered) require.NoError(t, err) // Verify the record is updated in the database @@ -1128,7 +1128,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) require.NoError(t, err) // Verify the record is deleted in the database @@ -1183,7 +1183,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) require.NoError(t, err) // Verify the record is deleted in the database @@ -1264,7 +1264,7 @@ func Test_workflowDeletedHandler(t *testing.T) { ) require.NoError(t, err) ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, active) + err = h.workflowRegisteredEvent(ctx, active, WorkflowRegistered) require.NoError(t, err) // Verify the record is updated in the database @@ -1283,7 +1283,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) require.Error(t, err, failWith) // Verify the record is still in the DB @@ -1298,6 +1298,7 @@ func Test_workflowDeletedHandler(t *testing.T) { type stubWorkflowArtifactsStore struct { spec *job.WorkflowSpec + upsertErr error deleteErr error deleteCalls atomic.Int32 } @@ -1314,6 +1315,9 @@ func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (* } func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(context.Context, *job.WorkflowSpec) (int64, error) { + if s.upsertErr != nil { + return 0, s.upsertErr + } return 1, nil } @@ -1342,7 +1346,7 @@ func Test_workflowDeletedEvent_DrainInProgress(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted) require.Error(t, err) require.ErrorIs(t, err, ErrDrainInProgress) assert.Equal(t, int32(1), drainable.drainCalls.Load()) @@ -1368,7 +1372,7 @@ func Test_workflowDeletedEvent_IgnoresErrAlreadyStopped(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted) require.NoError(t, err) assert.Equal(t, int32(1), drainable.closeCalls.Load()) assert.Equal(t, int32(1), artifactStore.deleteCalls.Load()) @@ -1406,7 +1410,7 @@ func Test_workflowRegisteredEvent_DrainingEngineNotTreatedAsHealthy(t *testing.T err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ Status: WorkflowStatusActive, WorkflowID: workflowID, - }) + }, WorkflowRegistered) require.Error(t, err) require.Contains(t, err.Error(), "could not clean up old engine") assert.Equal(t, int32(1), drainable.closeCalls.Load()) diff --git a/go.mod b/go.mod index be25ac97daa..1ecbda308c9 100644 --- a/go.mod +++ b/go.mod @@ -100,6 +100,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260623200841-e0322b819f62 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd @@ -430,6 +431,12 @@ require ( replace github.com/fbsobreira/gotron-sdk => github.com/smartcontractkit/chainlink-tron/relayer/gotron-sdk v0.0.5-0.20260218133534-cbd44da2856b +// Local replaces for the SHARED-2701 metering stack until chainlink-common and +// chainlink-protos/metering/go are tagged and published. +replace github.com/smartcontractkit/chainlink-common => ../chainlink-common + +replace github.com/smartcontractkit/chainlink-protos/metering/go => ../chainlink-protos/metering/go + tool github.com/smartcontractkit/chainlink-common/pkg/loop/cmd/loopinstall tool github.com/smartcontractkit/chainlink-common/script/cmd/dependabot diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 2882d359901..4a531844367 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -7,6 +7,44 @@ defaults: goflags: "-ldflags=-s" plugins: + cron: + - moduleURI: "github.com/smartcontractkit/capabilities/cron" + gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + installPath: "." + flags: "-tags timetzdata" + readcontract: + - moduleURI: "github.com/smartcontractkit/capabilities/readcontract" + gitRef: "3fad562d65996bf447372023b5123c0c6e8850c7" + installPath: "." + consensus: + - moduleURI: "github.com/smartcontractkit/capabilities/consensus" + gitRef: "69e882e0f24cf9ae897366c4dcbbd3c7e035ae33" + installPath: "." + workflowevent: + - enabled: false + moduleURI: "github.com/smartcontractkit/capabilities/workflowevent" + gitRef: "69e882e0f24cf9ae897366c4dcbbd3c7e035ae33" + installPath: "." + httpaction: + - moduleURI: "github.com/smartcontractkit/capabilities/http_action" + gitRef: "69e882e0f24cf9ae897366c4dcbbd3c7e035ae33" + installPath: "." + httptrigger: + - moduleURI: "github.com/smartcontractkit/capabilities/http_trigger" + gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + installPath: "." + evm: + - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/evm" + gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + installPath: "." + solana: + - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/solana" + gitRef: "3fccd7ffaa5662ce63d1cf081a9d65a9cd5a58d0" + installPath: "." + aptos: + - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/aptos" + gitRef: "3fad562d65996bf447372023b5123c0c6e8850c7" + installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" gitRef: "8812c6483a6e6cded22b3fe9615cf1d18b054fd5" From 276ddeea69a2e96daff4723de71e648795826a03 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 15 Jun 2026 14:58:21 -0400 Subject: [PATCH 02/14] bumping common and protos to remove local replace --- core/scripts/go.mod | 3 ++- core/scripts/go.sum | 6 ++++-- deployment/go.mod | 3 ++- deployment/go.sum | 6 ++++-- go.mod | 10 ++-------- go.sum | 6 ++++-- integration-tests/go.mod | 7 +++++-- integration-tests/go.sum | 6 ++++-- integration-tests/load/go.mod | 3 ++- integration-tests/load/go.sum | 6 ++++-- system-tests/lib/go.mod | 3 ++- system-tests/lib/go.sum | 6 ++++-- system-tests/tests/go.mod | 3 ++- system-tests/tests/go.sum | 6 ++++-- 14 files changed, 45 insertions(+), 29 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 2760670ca01..7352bd2ca4b 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -507,6 +507,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index cdbdafec702..afa95889506 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1580,8 +1580,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1626,6 +1626,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/deployment/go.mod b/deployment/go.mod index 3521ee2f1cb..d380046bfbd 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -440,6 +440,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 37355a0b094..3824cdbb1a7 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1385,8 +1385,8 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1431,6 +1431,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/go.mod b/go.mod index 1ecbda308c9..78218df496e 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a @@ -100,7 +100,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260623200841-e0322b819f62 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd @@ -431,12 +431,6 @@ require ( replace github.com/fbsobreira/gotron-sdk => github.com/smartcontractkit/chainlink-tron/relayer/gotron-sdk v0.0.5-0.20260218133534-cbd44da2856b -// Local replaces for the SHARED-2701 metering stack until chainlink-common and -// chainlink-protos/metering/go are tagged and published. -replace github.com/smartcontractkit/chainlink-common => ../chainlink-common - -replace github.com/smartcontractkit/chainlink-protos/metering/go => ../chainlink-protos/metering/go - tool github.com/smartcontractkit/chainlink-common/pkg/loop/cmd/loopinstall tool github.com/smartcontractkit/chainlink-common/script/cmd/dependabot diff --git a/go.sum b/go.sum index 7ac9b6f3c84..1d03191a326 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1204,6 +1204,8 @@ github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546- github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 2f4828c4fd7..efe2be9ea9a 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -33,7 +33,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae @@ -60,7 +60,10 @@ require ( gopkg.in/guregu/null.v4 v4.0.0 ) -require github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260706170510-94f3a0409ea5 // indirect +require ( + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect + github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260706170510-94f3a0409ea5 // indirect +) require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 8a628bc6bb3..09f39fdb0c3 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1372,8 +1372,8 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1418,6 +1418,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 0d2d85116d1..12f549a887e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.6-0.20260630120514-36abe27604df @@ -502,6 +502,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index c9737b0f1c1..ea97729af9b 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1634,8 +1634,8 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1680,6 +1680,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 6cb2227f59b..c6a8366bfd4 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260706100550-d43558069754 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae @@ -473,6 +473,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index b2c2cff2960..0829f3bbc41 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1545,8 +1545,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1591,6 +1591,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 85dcd38638e..d1b522c2d62 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -62,7 +62,7 @@ require ( github.com/rs/zerolog v1.35.1 github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -165,6 +165,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260706170510-94f3a0409ea5 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 3c65ac57379..7895b293a88 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1559,8 +1559,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0 h1:ywWOEZYL4DhVwf/TQ5jXsGlx6CzERsYbqd6ov3OS1sc= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260701091216-9264d4444ce0/go.mod h1:ncZiIgraMh4F9lWDoJwB1TX395qZFR5bvnZcHbD0A1Q= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1605,6 +1605,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= From 937a244acf69f7fc40354fb8b9d0958628b3b3fc Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 1 Jul 2026 17:17:29 -0400 Subject: [PATCH 03/14] updates from proto and plumbing config --- core/cmd/shell_local_test.go | 2 +- core/config/app_config.go | 1 + core/config/docs/core.toml | 20 ++++++ core/config/metering_config.go | 15 +++++ core/config/toml/types.go | 61 +++++++++++++++++++ core/services/chainlink/application.go | 2 +- core/services/chainlink/config_general.go | 4 ++ core/services/chainlink/config_metering.go | 58 ++++++++++++++++++ .../chainlink/config_metering_test.go | 41 +++++++++++++ core/services/chainlink/config_test.go | 9 +++ .../chainlink/mocks/general_config.go | 47 ++++++++++++++ .../testdata/config-empty-effective.toml | 9 +++ .../chainlink/testdata/config-full.toml | 9 +++ .../config-multi-chain-effective.toml | 9 +++ .../testdata/config-empty-effective.toml | 9 +++ core/web/resolver/testdata/config-full.toml | 9 +++ .../config-multi-chain-effective.toml | 9 +++ docs/CONFIG.md | 58 ++++++++++++++++++ plugins/loop_registry.go | 17 +++++- plugins/loop_registry_test.go | 19 ++++++ .../scripts/config/merge_raw_configs.txtar | 8 +++ testdata/scripts/node/validate/default.txtar | 8 +++ .../node/validate/defaults-override.txtar | 8 +++ .../disk-based-logging-disabled.txtar | 8 +++ .../validate/disk-based-logging-no-dir.txtar | 8 +++ .../node/validate/disk-based-logging.txtar | 8 +++ .../node/validate/fallback-override.txtar | 8 +++ .../node/validate/invalid-ocr-p2p.txtar | 8 +++ testdata/scripts/node/validate/invalid.txtar | 8 +++ testdata/scripts/node/validate/valid.txtar | 8 +++ testdata/scripts/node/validate/warnings.txtar | 8 +++ 31 files changed, 493 insertions(+), 3 deletions(-) create mode 100644 core/config/metering_config.go create mode 100644 core/services/chainlink/config_metering.go create mode 100644 core/services/chainlink/config_metering_test.go diff --git a/core/cmd/shell_local_test.go b/core/cmd/shell_local_test.go index 1682c8dbe16..6ad93300679 100644 --- a/core/cmd/shell_local_test.go +++ b/core/cmd/shell_local_test.go @@ -75,7 +75,7 @@ func genTestEVMRelayers(t *testing.T, cfg chainlink.GeneralConfig, ds sqlutil.Da f := chainlink.RelayerFactory{ Logger: lggr, LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), - cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), nil, "", cfg.LOOPP()), + cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), cfg.Metering(), nil, "", cfg.LOOPP()), CapabilitiesRegistry: capabilities.NewRegistry(lggr), } diff --git a/core/config/app_config.go b/core/config/app_config.go index 7965d096c73..9ace57f66d1 100644 --- a/core/config/app_config.go +++ b/core/config/app_config.go @@ -61,6 +61,7 @@ type AppConfig interface { WebServer() WebServer Tracing() Tracing Telemetry() Telemetry + Metering() Metering CRE() CRE CCV() CCV Billing() Billing diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 8a285a216e6..8c48e083d2e 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -922,6 +922,26 @@ Enabled = false # Default # By default, we only forward the go runtime metrics. Empty means forward everything. Prefixes = ["go_"] # Default +# Metering configures durable resource metering emission and the coarse +# deployment/node identity dimensions stamped on emitted MeterRecords and +# MeterSnapshots. These are plumbed to LOOP plugins via env. +[Metering] +# MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. +MeterRecordsEnabled = false # Default +# MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +# Requires MeterRecordsEnabled = true. +MeterSnapshotsEnabled = false # Default +# Product is the deployment product identity dimension, e.g. 'cre'. +Product = '' # Default +# Tenant is the deployment tenant identity dimension, e.g. 'mainline'. +Tenant = '' # Default +# Environment is the deployment environment identity dimension, e.g. 'production'. +Environment = '' # Default +# Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. +Zone = '' # Default +# NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). +NodeID = '' # Default + [CRE.Streams] # WsURL is the websockets url for the streams sdk config WsURL = "streams.url" # Example diff --git a/core/config/metering_config.go b/core/config/metering_config.go new file mode 100644 index 00000000000..6f3e3f53178 --- /dev/null +++ b/core/config/metering_config.go @@ -0,0 +1,15 @@ +package config + +// Metering exposes durable resource-metering configuration: the emission +// toggles and the coarse deployment/node identity dimensions stamped on emitted +// MeterRecords and MeterSnapshots. These are passed via loop.EnvConfig to every LOOP +// plugin. +type Metering interface { + MeterRecordsEnabled() bool + MeterSnapshotsEnabled() bool + Product() string + Tenant() string + Environment() string + Zone() string + NodeID() string +} diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 6b3bb050f1e..aff3a3c7fed 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -65,6 +65,7 @@ type Core struct { Mercury Mercury `toml:",omitempty"` Capabilities Capabilities `toml:",omitempty"` Telemetry Telemetry `toml:",omitempty"` + Metering Metering `toml:",omitempty"` Workflows Workflows `toml:",omitempty"` CRE CreConfig `toml:",omitempty"` Billing Billing `toml:",omitempty"` @@ -113,6 +114,7 @@ func (c *Core) SetFrom(f *Core) { c.JobDistributor.setFrom(&f.JobDistributor) c.Tracing.setFrom(&f.Tracing) c.Telemetry.setFrom(&f.Telemetry) + c.Metering.setFrom(&f.Metering) c.CRE.setFrom(&f.CRE) c.Billing.setFrom(&f.Billing) c.BridgeStatusReporter.setFrom(&f.BridgeStatusReporter) @@ -3084,6 +3086,65 @@ func (b *Telemetry) ValidateConfig() (err error) { return err } +// Metering configures durable resource metering emission and the coarse +// deployment/node identity dimensions stamped on emitted MeterRecords and +// MeterSnapshots. These are passed via loop.EnvConfig to every LOOP plugin. +type Metering struct { + // MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + MeterRecordsEnabled *bool + // MeterSnapshotsEnabled enables durable MeterSnapshot emission. Requires + // MeterRecordsEnabled to be true. + MeterSnapshotsEnabled *bool + // Product is the deployment product identity dimension, e.g. "cre". + Product *string + // Tenant is the deployment tenant identity dimension, e.g. "mainline". + Tenant *string + // Environment is the deployment environment dimension, e.g. "production". + Environment *string + // Zone is the deployment zone dimension, e.g. "wf-zone-a". + Zone *string + // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1" (NOT the CSA + // public key). The billing service uses it to look up the node's CSA key in + // the workflow registry; the CSA key itself is attached to events as the + // node_csa_key attribute. + NodeID *string +} + +func (b *Metering) setFrom(f *Metering) { + if v := f.MeterRecordsEnabled; v != nil { + b.MeterRecordsEnabled = v + } + if v := f.MeterSnapshotsEnabled; v != nil { + b.MeterSnapshotsEnabled = v + } + if v := f.Product; v != nil { + b.Product = v + } + if v := f.Tenant; v != nil { + b.Tenant = v + } + if v := f.Environment; v != nil { + b.Environment = v + } + if v := f.Zone; v != nil { + b.Zone = v + } + if v := f.NodeID; v != nil { + b.NodeID = v + } +} + +func (b *Metering) ValidateConfig() (err error) { + if b.MeterSnapshotsEnabled != nil && *b.MeterSnapshotsEnabled && (b.MeterRecordsEnabled == nil || !*b.MeterRecordsEnabled) { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "MeterSnapshotsEnabled", + Value: true, + Msg: "requires MeterRecordsEnabled to be true", + }) + } + return err +} + type PrometheusBridge struct { Enabled *bool Prefixes []string diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index a5ab27d7aed..fe5da2b1893 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -287,7 +287,7 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err } loopRegistry := plugins.NewLoopRegistry(globalLogger, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), - beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) + cfg.Metering(), beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) relayerFactory := RelayerFactory{ Logger: opts.Logger, diff --git a/core/services/chainlink/config_general.go b/core/services/chainlink/config_general.go index fd75e41e5bb..dcc99287d46 100644 --- a/core/services/chainlink/config_general.go +++ b/core/services/chainlink/config_general.go @@ -588,6 +588,10 @@ func (g *generalConfig) Telemetry() coreconfig.Telemetry { return &telemetryConfig{s: g.c.Telemetry} } +func (g *generalConfig) Metering() coreconfig.Metering { + return &meteringConfig{s: g.c.Metering} +} + func (g *generalConfig) CRE() coreconfig.CRE { return &creConfig{s: g.secrets.CRE, c: g.c.CRE} } diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go new file mode 100644 index 00000000000..28fa343c220 --- /dev/null +++ b/core/services/chainlink/config_metering.go @@ -0,0 +1,58 @@ +package chainlink + +import ( + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +type meteringConfig struct { + s toml.Metering +} + +func (b *meteringConfig) MeterRecordsEnabled() bool { + if b.s.MeterRecordsEnabled == nil { + return false + } + return *b.s.MeterRecordsEnabled +} + +func (b *meteringConfig) MeterSnapshotsEnabled() bool { + if b.s.MeterSnapshotsEnabled == nil { + return false + } + return *b.s.MeterSnapshotsEnabled +} + +func (b *meteringConfig) Product() string { + if b.s.Product == nil { + return "" + } + return *b.s.Product +} + +func (b *meteringConfig) Tenant() string { + if b.s.Tenant == nil { + return "" + } + return *b.s.Tenant +} + +func (b *meteringConfig) Environment() string { + if b.s.Environment == nil { + return "" + } + return *b.s.Environment +} + +func (b *meteringConfig) Zone() string { + if b.s.Zone == nil { + return "" + } + return *b.s.Zone +} + +func (b *meteringConfig) NodeID() string { + if b.s.NodeID == nil { + return "" + } + return *b.s.NodeID +} diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go new file mode 100644 index 00000000000..a9f0dcb4fda --- /dev/null +++ b/core/services/chainlink/config_metering_test.go @@ -0,0 +1,41 @@ +package chainlink + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +func TestMeteringConfig(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + mc := meteringConfig{s: toml.Metering{}} + assert.False(t, mc.MeterRecordsEnabled()) + assert.False(t, mc.MeterSnapshotsEnabled()) + assert.Empty(t, mc.Product()) + assert.Empty(t, mc.Tenant()) + assert.Empty(t, mc.Environment()) + assert.Empty(t, mc.Zone()) + assert.Empty(t, mc.NodeID()) + }) + + t.Run("explicit values", func(t *testing.T) { + mc := meteringConfig{s: toml.Metering{ + MeterRecordsEnabled: ptr(true), + MeterSnapshotsEnabled: ptr(true), + Product: ptr("cre"), + Tenant: ptr("mainline"), + Environment: ptr("production"), + Zone: ptr("wf-zone-a"), + NodeID: ptr("csa-pubkey-1"), + }} + assert.True(t, mc.MeterRecordsEnabled()) + assert.True(t, mc.MeterSnapshotsEnabled()) + assert.Equal(t, "cre", mc.Product()) + assert.Equal(t, "mainline", mc.Tenant()) + assert.Equal(t, "production", mc.Environment()) + assert.Equal(t, "wf-zone-a", mc.Zone()) + assert.Equal(t, "csa-pubkey-1", mc.NodeID()) + }) +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 27a27e70cc7..ae6c8876b8d 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -578,6 +578,15 @@ func TestConfig_Marshal(t *testing.T) { Prefixes: []string{"ocr_"}, }, } + full.Metering = toml.Metering{ + MeterRecordsEnabled: ptr(true), + MeterSnapshotsEnabled: ptr(true), + Product: ptr("cre"), + Tenant: ptr("mainline"), + Environment: ptr("production"), + Zone: ptr("wf-zone-a"), + NodeID: ptr("csa-pubkey-1"), + } full.CRE = toml.CreConfig{ UseLocalTimeProvider: ptr(true), EnableDKGRecipient: ptr(false), diff --git a/core/services/chainlink/mocks/general_config.go b/core/services/chainlink/mocks/general_config.go index 7bb1c4a4a1d..13228848409 100644 --- a/core/services/chainlink/mocks/general_config.go +++ b/core/services/chainlink/mocks/general_config.go @@ -2557,6 +2557,53 @@ func (_c *GeneralConfig_Telemetry_Call) RunAndReturn(run func() config.Telemetry return _c } +// Metering provides a mock function with no fields +func (_m *GeneralConfig) Metering() config.Metering { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Metering") + } + + var r0 config.Metering + if rf, ok := ret.Get(0).(func() config.Metering); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(config.Metering) + } + } + + return r0 +} + +// GeneralConfig_Metering_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Metering' +type GeneralConfig_Metering_Call struct { + *mock.Call +} + +// Metering is a helper method to define mock.On call +func (_e *GeneralConfig_Expecter) Metering() *GeneralConfig_Metering_Call { + return &GeneralConfig_Metering_Call{Call: _e.mock.On("Metering")} +} + +func (_c *GeneralConfig_Metering_Call) Run(run func()) *GeneralConfig_Metering_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *GeneralConfig_Metering_Call) Return(_a0 config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GeneralConfig_Metering_Call) RunAndReturn(run func() config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(run) + return _c +} + // TelemetryIngress provides a mock function with no fields func (_m *GeneralConfig) TelemetryIngress() config.TelemetryIngress { ret := _m.Called() diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 6aeec786180..17d6a7a44f0 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -364,6 +364,15 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Tenant = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index a2b99dafcbd..2501be075eb 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -403,6 +403,15 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'csa-pubkey-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index ec7cb65fbe0..8044f21d5db 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -364,6 +364,15 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Tenant = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 6aeec786180..17d6a7a44f0 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -364,6 +364,15 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Tenant = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index dc1cc00ae4d..561e87f011d 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -382,6 +382,15 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'csa-pubkey-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index a98ec27c405..9d85fae5486 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -364,6 +364,15 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Tenant = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/docs/CONFIG.md b/docs/CONFIG.md index af944ec92e5..1806ef7776d 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2566,6 +2566,64 @@ Prefixes = ["go_"] # Default Prefixes is a set of filters to restrict which prometheus metrics are forwarded based on prefix matching. By default, we only forward the go runtime metrics. Empty means forward everything. +## Metering +```toml +[Metering] +MeterRecordsEnabled = false # Default +MeterSnapshotsEnabled = false # Default +Product = '' # Default +Tenant = '' # Default +Environment = '' # Default +Zone = '' # Default +NodeID = '' # Default +``` +Metering configures durable resource metering emission and the coarse +deployment/node identity dimensions stamped on emitted MeterRecords and +MeterSnapshots. These are plumbed to every LOOP plugin via its environment. + +### MeterRecordsEnabled +```toml +MeterRecordsEnabled = false # Default +``` +MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + +### MeterSnapshotsEnabled +```toml +MeterSnapshotsEnabled = false # Default +``` +MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +Requires MeterRecordsEnabled = true. + +### Product +```toml +Product = '' # Default +``` +Product is the deployment product identity dimension, e.g. 'cre'. + +### Tenant +```toml +Tenant = '' # Default +``` +Tenant is the deployment tenant identity dimension, e.g. 'mainline'. + +### Environment +```toml +Environment = '' # Default +``` +Environment is the deployment environment identity dimension, e.g. 'production'. + +### Zone +```toml +Zone = '' # Default +``` +Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. + +### NodeID +```toml +NodeID = '' # Default +``` +NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). + ## CRE.Streams ```toml [CRE.Streams] diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index 3968018cdbc..158b8a46642 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -40,6 +40,7 @@ type LoopRegistry struct { autoPPROF config.AutoPprof cfgTracing config.Tracing cfgTelemetry config.Telemetry + cfgMetering config.Metering telemetryAuthHeaders map[string]string telemetryAuthPubKeyHex string cfgLOOPP config.LOOPP @@ -47,7 +48,7 @@ type LoopRegistry struct { func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, dbConfig config.Database, mercury de.Mercury, pyroscope config.Pyroscope, autoPPROF config.AutoPprof, tracing config.Tracing, telemetry config.Telemetry, - telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { + metering config.Metering, telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { return &LoopRegistry{ registry: map[string]*RegisteredLoop{}, lggr: logger.Named(lggr, "LoopRegistry"), @@ -59,6 +60,7 @@ func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, db autoPPROF: autoPPROF, cfgTracing: tracing, cfgTelemetry: telemetry, + cfgMetering: metering, telemetryAuthHeaders: telemetryAuthHeaders, telemetryAuthPubKeyHex: telemetryAuthPubKeyHex, cfgLOOPP: looppCfg, @@ -166,6 +168,19 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { envCfg.TelemetryPrometheusBridgeEnabled = m.cfgTelemetry.PrometheusBridge().Enabled() envCfg.TelemetryPrometheusBridgePrefixes = m.cfgTelemetry.PrometheusBridge().Prefixes() } + + // Metering config (emission toggles + deployment/node identity dimensions) + // is passed over the env channel to every LOOP plugin, rather than the + // standard-capabilities boundary. + if m.cfgMetering != nil { + envCfg.MeterRecordsEnabled = m.cfgMetering.MeterRecordsEnabled() + envCfg.MeterSnapshotsEnabled = m.cfgMetering.MeterSnapshotsEnabled() + envCfg.MeteringProduct = m.cfgMetering.Product() + envCfg.MeteringTenant = m.cfgMetering.Tenant() + envCfg.MeteringEnvironment = m.cfgMetering.Environment() + envCfg.MeteringZone = m.cfgMetering.Zone() + envCfg.MeteringNodeID = m.cfgMetering.NodeID() + } m.lggr.Debugf("Registered loopp %q with port %d", id, envCfg.PrometheusPort) // Add auth header after logging config diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index d762dd85c57..80216f0e7f5 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -92,6 +92,16 @@ func (m mockCfgTelemetry) PrometheusBridge() config.PrometheusBridge { return mockPrometheusBridge{} } +type mockCfgMetering struct{} + +func (m mockCfgMetering) MeterRecordsEnabled() bool { return true } +func (m mockCfgMetering) MeterSnapshotsEnabled() bool { return true } +func (m mockCfgMetering) Product() string { return "cre" } +func (m mockCfgMetering) Tenant() string { return "mainline" } +func (m mockCfgMetering) Environment() string { return "production" } +func (m mockCfgMetering) Zone() string { return "wf-zone-a" } +func (m mockCfgMetering) NodeID() string { return "csa-pubkey-1" } + type mockPrometheusBridge struct{} func (m mockPrometheusBridge) Enabled() bool { return true } @@ -185,6 +195,7 @@ func TestLoopRegistry_Register(t *testing.T) { mockCfgMercury := &mockCfgMercury{} mockCfgTracing := &mockCfgTracing{} mockCfgTelemetry := &mockCfgTelemetry{} + mockCfgMetering := &mockCfgMetering{} registry := make(map[string]*RegisteredLoop) // Create a LoopRegistry instance with mockCfgTracing @@ -197,6 +208,7 @@ func TestLoopRegistry_Register(t *testing.T) { cfgMercury: mockCfgMercury, cfgTracing: mockCfgTracing, cfgTelemetry: mockCfgTelemetry, + cfgMetering: mockCfgMetering, } // Test case 1: Register new loop @@ -250,6 +262,13 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, 512, envCfg.TelemetryLogExportMaxBatchSize) require.Equal(t, 5*time.Second, envCfg.TelemetryLogExportInterval) require.Equal(t, 2048, envCfg.TelemetryLogMaxQueueSize) + require.True(t, envCfg.MeterRecordsEnabled) + require.True(t, envCfg.MeterSnapshotsEnabled) + require.Equal(t, "cre", envCfg.MeteringProduct) + require.Equal(t, "mainline", envCfg.MeteringTenant) + require.Equal(t, "production", envCfg.MeteringEnvironment) + require.Equal(t, "wf-zone-a", envCfg.MeteringZone) + require.Equal(t, "csa-pubkey-1", envCfg.MeteringNodeID) require.Equal(t, "example.com/chip-ingress", envCfg.ChipIngressEndpoint) require.False(t, envCfg.ChipIngressBatchEmitterEnabled) diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 57977a1f485..3b74face983 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -511,6 +511,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index c0a1abab84b..9e0f05c374c 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -376,6 +376,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 86086a4fca2..5e7958e30a7 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -437,6 +437,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 36c2900e57f..a6aec07ebde 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -420,6 +420,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 9fb002e0af8..47e3d90fc7a 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -420,6 +420,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index dfdb9b48384..37ed971264b 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -420,6 +420,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index c2320cbce92..629b75c1c10 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -522,6 +522,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index 5e7eaaa1bad..488f4199388 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -405,6 +405,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index aae5e8883ca..cec424c990c 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -416,6 +416,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 9b0595833cb..c8b9da305ce 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -417,6 +417,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 6e000e852ba..32598b0d36e 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -399,6 +399,14 @@ LogMaxQueueSize = 2048 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 From 8fd5251ffba99a4da4e0e15e338b640a6dd15e14 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Tue, 7 Jul 2026 13:45:06 -0400 Subject: [PATCH 04/14] adding numericTenantID + reducing naming complexity in protos --- core/config/docs/core.toml | 4 +- core/config/metering_config.go | 1 + core/config/toml/types.go | 7 +- core/services/chainlink/config_metering.go | 7 ++ .../chainlink/config_metering_test.go | 3 + core/services/chainlink/config_test.go | 1 + .../testdata/config-empty-effective.toml | 1 + .../chainlink/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 1 + .../cre/confidential_relay_peerid_test.go | 1 + core/services/cre/cre.go | 74 +++++------- .../services/standardcapabilities/delegate.go | 17 +-- .../standard_capabilities.go | 19 +--- .../standard_capabilities_test.go | 24 +--- core/services/workflows/syncer/v2/handler.go | 107 +++++++++++------- .../syncer/v2/handler_metering_test.go | 60 +++++----- .../testdata/config-empty-effective.toml | 1 + core/web/resolver/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 1 + docs/CONFIG.md | 11 +- plugins/loop_registry.go | 11 +- plugins/loop_registry_test.go | 12 +- 22 files changed, 180 insertions(+), 185 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 8c48e083d2e..f4e389b92b9 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -933,8 +933,10 @@ MeterRecordsEnabled = false # Default MeterSnapshotsEnabled = false # Default # Product is the deployment product identity dimension, e.g. 'cre'. Product = '' # Default -# Tenant is the deployment tenant identity dimension, e.g. 'mainline'. +# Tenant is the human-readable tenant name, e.g. 'mainline'. Tenant = '' # Default +# NumericTenantID is the numbered tenant identifier represented as a string. +NumericTenantID = '' # Default # Environment is the deployment environment identity dimension, e.g. 'production'. Environment = '' # Default # Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. diff --git a/core/config/metering_config.go b/core/config/metering_config.go index 6f3e3f53178..9f11e1c3419 100644 --- a/core/config/metering_config.go +++ b/core/config/metering_config.go @@ -9,6 +9,7 @@ type Metering interface { MeterSnapshotsEnabled() bool Product() string Tenant() string + NumericTenantID() string Environment() string Zone() string NodeID() string diff --git a/core/config/toml/types.go b/core/config/toml/types.go index aff3a3c7fed..6111804b62f 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -3097,8 +3097,10 @@ type Metering struct { MeterSnapshotsEnabled *bool // Product is the deployment product identity dimension, e.g. "cre". Product *string - // Tenant is the deployment tenant identity dimension, e.g. "mainline". + // Tenant is the human-readable tenant name, e.g. "mainline". Tenant *string + // NumericTenantID is the numbered tenant identifier as a string. + NumericTenantID *string // Environment is the deployment environment dimension, e.g. "production". Environment *string // Zone is the deployment zone dimension, e.g. "wf-zone-a". @@ -3123,6 +3125,9 @@ func (b *Metering) setFrom(f *Metering) { if v := f.Tenant; v != nil { b.Tenant = v } + if v := f.NumericTenantID; v != nil { + b.NumericTenantID = v + } if v := f.Environment; v != nil { b.Environment = v } diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go index 28fa343c220..fa0c8f36843 100644 --- a/core/services/chainlink/config_metering.go +++ b/core/services/chainlink/config_metering.go @@ -36,6 +36,13 @@ func (b *meteringConfig) Tenant() string { return *b.s.Tenant } +func (b *meteringConfig) NumericTenantID() string { + if b.s.NumericTenantID == nil { + return "" + } + return *b.s.NumericTenantID +} + func (b *meteringConfig) Environment() string { if b.s.Environment == nil { return "" diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go index a9f0dcb4fda..56d61af1925 100644 --- a/core/services/chainlink/config_metering_test.go +++ b/core/services/chainlink/config_metering_test.go @@ -15,6 +15,7 @@ func TestMeteringConfig(t *testing.T) { assert.False(t, mc.MeterSnapshotsEnabled()) assert.Empty(t, mc.Product()) assert.Empty(t, mc.Tenant()) + assert.Empty(t, mc.NumericTenantID()) assert.Empty(t, mc.Environment()) assert.Empty(t, mc.Zone()) assert.Empty(t, mc.NodeID()) @@ -26,6 +27,7 @@ func TestMeteringConfig(t *testing.T) { MeterSnapshotsEnabled: ptr(true), Product: ptr("cre"), Tenant: ptr("mainline"), + NumericTenantID: ptr("42"), Environment: ptr("production"), Zone: ptr("wf-zone-a"), NodeID: ptr("csa-pubkey-1"), @@ -34,6 +36,7 @@ func TestMeteringConfig(t *testing.T) { assert.True(t, mc.MeterSnapshotsEnabled()) assert.Equal(t, "cre", mc.Product()) assert.Equal(t, "mainline", mc.Tenant()) + assert.Equal(t, "42", mc.NumericTenantID()) assert.Equal(t, "production", mc.Environment()) assert.Equal(t, "wf-zone-a", mc.Zone()) assert.Equal(t, "csa-pubkey-1", mc.NodeID()) diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index ae6c8876b8d..a5e654065e9 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -583,6 +583,7 @@ func TestConfig_Marshal(t *testing.T) { MeterSnapshotsEnabled: ptr(true), Product: ptr("cre"), Tenant: ptr("mainline"), + NumericTenantID: ptr("42"), Environment: ptr("production"), Zone: ptr("wf-zone-a"), NodeID: ptr("csa-pubkey-1"), diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 17d6a7a44f0..cb9ed7589f7 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -369,6 +369,7 @@ MeterRecordsEnabled = false MeterSnapshotsEnabled = false Product = '' Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 2501be075eb..907bb21bc98 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -408,6 +408,7 @@ MeterRecordsEnabled = true MeterSnapshotsEnabled = true Product = 'cre' Tenant = 'mainline' +NumericTenantID = '42' Environment = 'production' Zone = 'wf-zone-a' NodeID = 'csa-pubkey-1' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 8044f21d5db..d42d768567e 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -369,6 +369,7 @@ MeterRecordsEnabled = false MeterSnapshotsEnabled = false Product = '' Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/core/services/cre/confidential_relay_peerid_test.go b/core/services/cre/confidential_relay_peerid_test.go index 77f376a5d74..47692762125 100644 --- a/core/services/cre/confidential_relay_peerid_test.go +++ b/core/services/cre/confidential_relay_peerid_test.go @@ -52,6 +52,7 @@ func (s stubConfig) CRE() config.CRE { return nil } func (s stubConfig) P2P() config.P2P { return s.p2p } func (s stubConfig) Sharding() config.Sharding { return nil } func (s stubConfig) Telemetry() config.Telemetry { return nil } +func (s stubConfig) Metering() config.Metering { return nil } func peerIDFromByte(b byte) p2pkey.PeerID { var id p2pkey.PeerID diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index c57a247cd9f..e3da8377357 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -300,9 +300,9 @@ func (s *Services) newSubservices( return srvs, nil } - // Build the syncer's base metering identity once from node config + CSA key; + // Build the syncer's base metering identity once from node metering config; // the handler resolves the workflow DON id later from the don notifier. - meterIdentity := newSyncerMeterIdentity(cfg, keyStore, lggr) + meterIdentity := newSyncerMeterIdentity(cfg) wfSyncer, billingClient, wfSyncerSrvcs, err := newWorkflowRegistrySyncer( cfg, @@ -347,6 +347,7 @@ type Config interface { P2P() config.P2P Sharding() config.Sharding Telemetry() config.Telemetry + Metering() config.Metering } // RelayerChainInterops is the minimal interface needed for relayer chain interops @@ -727,47 +728,28 @@ func newBillingClient(lggr logger.Logger, cfg Config, opts Opts) (metering.Billi return billing.NewWorkflowClient(lggr, cfg.Billing().URL(), workflowOpts...) } -// Metering identity sourcing constants. meterProduct is the deployment product -// dimension for all CRE metering records. The environment/zone dimensions are -// read from the node's [Telemetry.ResourceAttributes] map under these keys, the -// same map the host injects into trigger LOOPs (see core/services/ -// standardcapabilities), so a node's engine and its trigger LOOPs agree on the -// coarse identity. -const ( - meterProduct = "cre" - resourceAttrEnvKey = "env" - resourceAttrZoneKey = "zone" -) - -// nodeCSAPublicKeyHex returns the node's CSA public key as hex — the canonical -// node_id used uniformly across a node's engine and its trigger LOOPs (matching -// keystore.BuildBeholderAuth, which derives the beholder auth pubkey from the -// same default CSA key). A node has at most one CSA key; an empty string is -// returned (with a warning) when none is available, so metering degrades to an -// empty node_id rather than failing node startup. -func nodeCSAPublicKeyHex(keyStore Keystore, lggr logger.Logger) string { - keys, err := keyStore.CSA().GetAll() - if err != nil || len(keys) == 0 { - lggr.Warnw("no CSA key available for metering node_id; node_id will be empty", "err", err) - return "" - } - return keys[0].PublicKeyString() -} - // newSyncerMeterIdentity builds the syncer's base metering identity from node -// config: product is the constant, environment/zone come from -// [Telemetry.ResourceAttributes], and node_id is the CSA public key. The -// workflow DON id (don_id) is resolved later by the handler from the don -// notifier (the engine runs on the workflow DON), so it is intentionally left -// empty here. Service/Resource/ResourceType are stamped by WithIdentity. -func newSyncerMeterIdentity(cfg Config, keyStore Keystore, lggr logger.Logger) resourcemanager.ResourceIdentity { - attrs := cfg.Telemetry().ResourceAttributes() - return resourcemanager.ResourceIdentity{ - Product: meterProduct, - Environment: attrs[resourceAttrEnvKey], - Zone: attrs[resourceAttrZoneKey], - NodeID: nodeCSAPublicKeyHex(keyStore, lggr), - } +// metering config. Product/tenant/environment/zone and logical node_id are read +// from [Metering]. The workflow DON id (don_id) is resolved later by the +// handler from the don notifier (the engine runs on the workflow DON), so it is +// intentionally left empty here. Service/resource_pool are stamped by +// syncer.WithIdentity. +func newSyncerMeterIdentity(cfg Config) resourcemanager.ResourceIdentity { + m := cfg.Metering() + if m == nil { + return resourcemanager.ResourceIdentity{} + } + id := resourcemanager.ResourceIdentity{ + Product: m.Product(), + Tenant: m.Tenant(), + NumericTenantID: m.NumericTenantID(), + Environment: m.Environment(), + Zone: m.Zone(), + } + if nodeID := m.NodeID(); nodeID != "" { + id.Don = &resourcemanager.DonIdentity{NodeID: nodeID} + } + return id } // meterRecordsEnabled reads the CL_METER_RECORDS_ENABLED env var gating emission of @@ -1079,6 +1061,7 @@ func newWorkflowRegistrySyncerV2( } shardRoutingSteady = shardownership.NewSteadySignal(shardownership.WithSteadySignalMetrics(steadyMetrics)) } + meterRecords := meterRecordsEnabled(lggr) handlerOpts := []syncerV2.EventHandlerOption{ syncerV2.WithBillingClient(billingClient), @@ -1093,9 +1076,10 @@ func newWorkflowRegistrySyncerV2( // SnapshotInterval enables the periodic absolute-state snapshot loop; the // RM is otherwise a no-op when metering is disabled. syncerV2.WithResourceManager(resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ - Enabled: meterRecordsEnabled(lggr), - Emitter: beholder.GetEmitter(), - SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + MeterRecordsEnabled: meterRecords, + MeterSnapshotsEnabled: meterRecords, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, })), syncerV2.WithIdentity(meterIdentity), } diff --git a/core/services/standardcapabilities/delegate.go b/core/services/standardcapabilities/delegate.go index 9cd5e7275be..7282ff6e1cf 100644 --- a/core/services/standardcapabilities/delegate.go +++ b/core/services/standardcapabilities/delegate.go @@ -46,12 +46,9 @@ type RelayGetter interface { GetIDToRelayerMap() map[types.RelayID]loop.Relayer } -// NodeIdentity is the host-injected deployment/node metering identity that the -// Delegate stamps onto every spawned capability's StandardCapabilitiesDependencies, -// mirroring how the engine sources the same dimensions from node config (see -// core/services/cre). It is sourced once at node startup (where the CSA key and -// telemetry config are both available) so operators configure it in one place and -// a node's engine and its trigger LOOPs agree on product/environment/zone/node_id. +// NodeIdentity is the host-injected deployment/node metering identity sourced at +// node startup. It is retained on the delegate for compatibility with launcher +// wiring while metering identity delivery is handled through loop.EnvConfig. type NodeIdentity struct { // Product is the deployment product, e.g. "cre". Product string @@ -59,7 +56,7 @@ type NodeIdentity struct { Environment string // Zone is the deployment zone, from [Telemetry.ResourceAttributes]["zone"]. Zone string - // NodeID is the node's CSA public key (hex), matching the engine's node_id. + // NodeID is the node's logical name (not the CSA public key). NodeID string } @@ -428,12 +425,6 @@ func (d *Delegate) NewServices( CRESettings: d.creSettings, TriggerEventStore: triggercap.NewTriggerEventStore(d.ds), CapabilityDonID: capabilityDonID, - // Host-injected deployment/node metering identity, delivered to trigger - // LOOPs through the standardized Initialise channel. - Product: d.nodeIdentity.Product, - Environment: d.nodeIdentity.Environment, - Zone: d.nodeIdentity.Zone, - NodeID: d.nodeIdentity.NodeID, } standardCapability := NewStandardCapabilities(log, command, configJSON, d.cfg, dependencies) diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index bc7f600e24c..01ea074dc44 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -46,13 +46,6 @@ type StandardCapabilities struct { // at Initialise time. Zero means the host did not resolve one; the plugin // will fall back to capability-registry lookup. capabilityDonID uint32 - // Host-injected deployment/node metering identity, captured from the deps - // passed at construction and re-delivered to the LOOP through the deps built - // for Initialise below. - product string - environment string - zone string - nodeID string capabilitiesLoop *loop.StandardCapabilitiesService @@ -84,19 +77,13 @@ func NewStandardCapabilities( creSettings: dependencies.CRESettings, triggerEventStore: dependencies.TriggerEventStore, capabilityDonID: dependencies.CapabilityDonID, - product: dependencies.Product, - environment: dependencies.Environment, - zone: dependencies.Zone, - nodeID: dependencies.NodeID, stopChan: make(chan struct{}), readyChan: make(chan struct{}), } } // initialiseDependencies builds the StandardCapabilitiesDependencies delivered to -// the capability LOOP via Initialise. It re-emits the host-injected metering -// identity (product/environment/zone/node_id) captured at construction so trigger -// LOOPs receive it through the standardized Initialise channel. +// the capability LOOP via Initialise. func (s *StandardCapabilities) initialiseDependencies() core.StandardCapabilitiesDependencies { return core.StandardCapabilitiesDependencies{ Config: s.config, @@ -110,10 +97,6 @@ func (s *StandardCapabilities) initialiseDependencies() core.StandardCapabilitie CRESettings: s.creSettings, TriggerEventStore: s.triggerEventStore, CapabilityDonID: s.capabilityDonID, - Product: s.product, - Environment: s.environment, - Zone: s.zone, - NodeID: s.nodeID, } } diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index 622166a54be..3884c702d3c 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -134,38 +134,22 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { }) } -// TestStandardCapabilities_ForwardsNodeMeteringIdentity asserts that the -// host-injected deployment/node metering identity (Product/Environment/Zone/ -// NodeID) supplied on the deps at construction is re-delivered, unchanged, on the -// StandardCapabilitiesDependencies handed to the capability LOOP at Initialise. -// This is the standardized Initialise channel that gives trigger LOOPs the same -// coarse metering identity the node's engine uses. -func TestStandardCapabilities_ForwardsNodeMeteringIdentity(t *testing.T) { +func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { want := core.StandardCapabilitiesDependencies{ - Product: "cre", - Environment: "staging", - Zone: "wf-zone-a", - NodeID: "0a1b2c3d4e5f", + Config: "test-config", } std := NewStandardCapabilities( logger.TestLogger(t), "not/found/path/to/binary", - "{}", + want.Config, &capturingRegistrar{}, want, ) got := std.initialiseDependencies() - require.Equal(t, want.Product, got.Product, - "the host-injected product must reach the capability LOOP via Initialise") - require.Equal(t, want.Environment, got.Environment, - "the host-injected environment must reach the capability LOOP via Initialise") - require.Equal(t, want.Zone, got.Zone, - "the host-injected zone must reach the capability LOOP via Initialise") - require.Equal(t, want.NodeID, got.NodeID, - "the host-injected node_id (CSA pubkey) must reach the capability LOOP via Initialise") + require.Equal(t, want.Config, got.Config, "config should be re-delivered to LOOP Initialise dependencies") } const capturingRegistrarErr = "capturingRegistrar: stop after capture" diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 6c871f881bc..ea7e27d8d89 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -69,13 +69,14 @@ type DrainableService interface { var ErrDrainInProgress = errors.New("drain in progress") // Service-level metering identity constants for the workflow syncer. Service is -// the stable service constant; Resource/ResourceType identify the workflow_specs_v2 -// pool and its billing unit. The coarse deployment/node/DON dimensions (product, -// environment, zone, don_id, node_id) are supplied at construction via -// WithIdentity; resource_id is set per workflow via WithResourceID. +// the stable service constant; ResourcePool identifies the workflow_specs_v2 +// pool. The billing unit (ResourceType) and per-resource id (ResourceID) are +// carried on each Utilization. The coarse deployment/node/DON dimensions +// (product, tenant, environment, zone, don_id, node_id) are supplied at +// construction via WithIdentity. const ( meterService = "workflow-syncer-v2" - meterResource = "workflow_specs_v2" + meterResourcePool = "workflow_specs_v2" meterResourceType = "operations" ) @@ -105,11 +106,12 @@ type eventHandler struct { billingClient metering.BillingClient resourceManager *resourcemanager.ResourceManager // meterIdentity is the base metering identity for this node's syncer: the - // six coarse dimensions (product, environment, zone, don_id, node_id, - // service) plus the workflow_specs_v2 resource/resource_type. resource_id is - // set per workflow via WithResourceID. It is populated by WithIdentity in - // cre.go from node TOML + CSA + the workflow DON; Service/Resource/ - // ResourceType are forced to the syncer constants regardless of the option. + // six coarse dimensions (product, tenant, environment, zone, don_id, + // node_id) plus service/resource_pool. Per-resource billing fields + // (resource_type/resource_id/event_id/org_id/value) are attached at emit/ + // snapshot time via UtilizationFields. It is populated by WithIdentity in + // cre.go; Service/ResourcePool are forced to syncer constants regardless of + // the option. meterIdentity resourcemanager.ResourceIdentity // resolvedDonID holds the workflow DON id once resolved from the don notifier // at start (the engine runs on the workflow DON). It is resolved @@ -194,17 +196,14 @@ func WithResourceManager(rm *resourcemanager.ResourceManager) func(*eventHandler } // WithIdentity supplies the coarse metering identity dimensions sourced once at -// construction in cre.go (product, environment, zone from node TOML; node_id = -// CSA pubkey hex; don_id = the workflow DON the engine runs on). The syncer -// Service/Resource/ResourceType are stable constants and always overwrite -// whatever the caller passes, so the option carries only the deployment/node/DON -// dimensions; resource_id is left empty and set per workflow via WithResourceID. +// construction in cre.go (product/tenant/environment/zone/node_id from node +// metering config; don_id resolved later by the handler). The syncer +// Service/ResourcePool are stable constants and always overwrite whatever the +// caller passes. func WithIdentity(id resourcemanager.ResourceIdentity) func(*eventHandler) { return func(e *eventHandler) { id.Service = meterService - id.Resource = meterResource - id.ResourceType = meterResourceType - id.ResourceID = "" + id.ResourcePool = meterResourcePool e.meterIdentity = id } } @@ -367,8 +366,7 @@ func NewEventHandler( // deployment/node/DON dimensions are filled in by WithIdentity in cre.go. meterIdentity: resourcemanager.ResourceIdentity{ Service: meterService, - Resource: meterResource, - ResourceType: meterResourceType, + ResourcePool: meterResourcePool, }, tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode } @@ -1015,9 +1013,9 @@ func (h *eventHandler) workflowDeletedEvent( // emitMeterRecord emits a metering.v1.MeterRecord for a workflow_specs_v2 mutation. // Callers must invoke it only after the database mutation has succeeded. Emission is -// fail-open: it never returns an error and must never affect event handling. The -// idempotency key is deterministic over workflowID, originatingEvent, and action, so -// a retried event emits a record the billing consumer dedups against the original. +// fail-open: it never returns an error and must never affect event handling. +// Consumer-side dedup can use the emitted identity + action + +// utilization(resource_id/event_id). // // Known lost-record windows (consumer-side reconciliation tracked by SHARED-2141): // - A node crash between the workflow spec database write and this emit loses the @@ -1030,12 +1028,18 @@ func (h *eventHandler) emitMeterRecord(ctx context.Context, action meteringpb.Me if h.resourceManager == nil { return } - // resource_id = workflow_id (the syncer has no shared physical resource); the - // originating event name is the event identity that distinguishes lifecycle - // edges in the idempotency key. - identity := h.baseIdentity().WithResourceID(workflowID) + // resource_id = workflow_id (the syncer has no shared physical resource); + // event_id = originating event name so lifecycle edges remain distinguishable. + identity := h.baseIdentity() + utilizations := []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + EventID: string(originatingEvent), + }), + } h.resourceManager.EmitMeterRecord(ctx, identity, action, - resourcemanager.NewUtilization(identity, action, 1, string(originatingEvent))) + utilizations) } // baseIdentity returns the handler's metering identity with the workflow DON id @@ -1045,25 +1049,33 @@ func (h *eventHandler) emitMeterRecord(ctx context.Context, action meteringpb.Me func (h *eventHandler) baseIdentity() resourcemanager.ResourceIdentity { id := h.meterIdentity if donID := h.resolvedDonID.Load(); donID != nil { - id.DONID = *donID + nodeID := "" + if id.Don != nil { + nodeID = id.Don.NodeID + } + id.Don = &resourcemanager.DonIdentity{ + DonID: *donID, + NodeID: nodeID, + } } return id } // ResourceIdentity implements resourcemanager.Meterable: it returns the syncer's -// base identity (six coarse dimensions + workflow_specs_v2 resource/resource_type, -// resource_id empty). The ResourceManager uses it as the top-level identity of -// each emitted Snapshot. +// base identity (coarse deployment/DON/node dimensions + service/resource_pool). +// The ResourceManager uses it as the top-level identity of each emitted +// Snapshot. func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { return h.baseIdentity() } // GetUtilization implements resourcemanager.Meterable: it returns one -// SnapshotEntry per running engine, with Value 1 (each running workflow holds one -// workflow_specs_v2 reservation). It is a pure in-memory read over the engine -// registry — each entry's resource_id is the workflow_id, which fully identifies -// the resource (Design 1; reconciling persisted-but-not-running specs is the -// Design 2 follow-up, see snapshotDesign2Followup below). +// SnapshotEntry per running engine, each with one Utilization at value 1 +// (each running workflow holds one workflow_specs_v2 reservation). It is a pure +// in-memory read over the engine registry — each entry's resource_id is the +// workflow_id, which fully identifies the resource (Design 1; reconciling +// persisted-but-not-running specs is the Design 2 follow-up, see +// snapshotDesign2Followup below). func (h *eventHandler) GetUtilization(_ context.Context) []resourcemanager.SnapshotEntry { base := h.baseIdentity() engines := h.engineRegistry.GetAll() @@ -1071,8 +1083,13 @@ func (h *eventHandler) GetUtilization(_ context.Context) []resourcemanager.Snaps for _, e := range engines { workflowID := e.WorkflowID.Hex() entries = append(entries, resourcemanager.SnapshotEntry{ - Identity: base.WithResourceID(workflowID), - Value: 1, + Identity: base, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + }), + }, }) } return entries @@ -1098,12 +1115,18 @@ func (h *eventHandler) emitGracefulCloseReleases(ctx context.Context, engines [] if h.resourceManager == nil { return } - base := h.baseIdentity() + identity := h.baseIdentity() for _, e := range engines { workflowID := e.WorkflowID.Hex() - identity := base.WithResourceID(workflowID) + utilizations := []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + EventID: string(WorkflowDeleted), + }), + } h.resourceManager.EmitMeterRecord(ctx, identity, meteringpb.MeterAction_METER_ACTION_RELEASE, - resourcemanager.NewUtilization(identity, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, string(WorkflowDeleted))) + utilizations) } } diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index d48ed3d90fc..9f31cc5b03b 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -66,8 +66,8 @@ func (r *recordingEmitter) Records() []*meteringpb.MeterRecord { func newMeteringResourceManager(t *testing.T, enabled bool, emitter resourcemanager.Emitter) *resourcemanager.ResourceManager { t.Helper() return resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ - Enabled: enabled, - Emitter: emitter, + MeterRecordsEnabled: enabled, + Emitter: emitter, }) } @@ -137,29 +137,16 @@ func requireMeterRecord(t *testing.T, record *meteringpb.MeterRecord, action met t.Helper() require.NotNil(t, record.Identity) assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) - assert.Equal(t, "workflow_specs_v2", record.Identity.Resource) - assert.Equal(t, "operations", record.Identity.ResourceType) - // resource_id = workflow_id for the syncer (no shared physical resource). - assert.Equal(t, workflowID, record.Identity.ResourceId) + assert.Equal(t, "workflow_specs_v2", record.Identity.ResourcePool) assert.Equal(t, action, record.Action) assert.NotNil(t, record.Timestamp) - require.NotNil(t, record.Utilization) - assert.Equal(t, int64(1), record.Utilization.Value) - // The idempotency key is derived over the full identity (with resource_id set - // to workflow_id) and the originating event name as the event identity. - wantIdentity := record.Identity - wantID := resourcemanager.ResourceIdentity{ - Product: wantIdentity.Product, - Environment: wantIdentity.Environment, - Zone: wantIdentity.Zone, - DONID: wantIdentity.DonId, - NodeID: wantIdentity.NodeId, - Service: wantIdentity.Service, - Resource: wantIdentity.Resource, - ResourceType: wantIdentity.ResourceType, - ResourceID: wantIdentity.ResourceId, - } - assert.Equal(t, resourcemanager.IdempotencyKey(wantID, action, string(originatingEvent)), record.Utilization.IdempotencyKey) + require.Len(t, record.Utilizations, 1) + util := record.Utilizations[0] + assert.Equal(t, "1", util.Value) + assert.Equal(t, "operations", util.ResourceType) + // resource_id = workflow_id for the syncer (no shared physical resource). + assert.Equal(t, workflowID, util.ResourceId) + assert.Equal(t, string(originatingEvent), util.EventId) } func Test_meterRecords(t *testing.T) { @@ -187,7 +174,7 @@ func Test_meterRecords(t *testing.T) { requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RESERVE, WorkflowRegistered, wfID.Hex()) }) - t.Run("retried registered event emits an identical idempotency key", func(t *testing.T) { + t.Run("retried registered event emits equivalent utilization identity", func(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} // The stub never returns a stored spec, so each call replays the @@ -205,7 +192,13 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 2) - assert.Equal(t, records[0].Utilization.IdempotencyKey, records[1].Utilization.IdempotencyKey) + require.Len(t, records[0].Utilizations, 1) + require.Len(t, records[1].Utilizations, 1) + assert.Equal(t, records[0].Action, records[1].Action) + assert.Equal(t, records[0].Identity.GetService(), records[1].Identity.GetService()) + assert.Equal(t, records[0].Identity.GetResourcePool(), records[1].Identity.GetResourcePool()) + assert.Equal(t, records[0].Utilizations[0].GetResourceId(), records[1].Utilizations[0].GetResourceId()) + assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) }) t.Run("activated event with existing spec emits UPDATE", func(t *testing.T) { @@ -404,10 +397,11 @@ func Test_meterRecords(t *testing.T) { emitter := &recordingSnapshotEmitter{} clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) rm := resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, }) h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, rm) unregister := rm.Register(h) @@ -434,11 +428,11 @@ func Test_meterRecords(t *testing.T) { for _, snap := range snapshots { require.NotNil(t, snap.Identity) assert.Equal(t, "workflow-syncer-v2", snap.Identity.Service) - assert.Equal(t, "workflow_specs_v2", snap.Identity.Resource) - require.NotNil(t, snap.Utilization) - assert.Equal(t, int64(1), snap.Utilization.Value) + assert.Equal(t, "workflow_specs_v2", snap.Identity.ResourcePool) + require.Len(t, snap.Utilization, 1) + assert.Equal(t, "1", snap.Utilization[0].Value) // resource_id = workflow_id fully identifies the resource; no labels. - byWorkflowID[snap.Identity.ResourceId] = snap + byWorkflowID[snap.Utilization[0].ResourceId] = snap } require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first running engine") require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second running engine") diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 17d6a7a44f0..cb9ed7589f7 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -369,6 +369,7 @@ MeterRecordsEnabled = false MeterSnapshotsEnabled = false Product = '' Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 561e87f011d..66a6e5a19fd 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -387,6 +387,7 @@ MeterRecordsEnabled = true MeterSnapshotsEnabled = true Product = 'cre' Tenant = 'mainline' +NumericTenantID = '42' Environment = 'production' Zone = 'wf-zone-a' NodeID = 'csa-pubkey-1' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 9d85fae5486..c7a8e8dc3eb 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -369,6 +369,7 @@ MeterRecordsEnabled = false MeterSnapshotsEnabled = false Product = '' Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 1806ef7776d..b5a8cb50c0d 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2573,13 +2573,14 @@ MeterRecordsEnabled = false # Default MeterSnapshotsEnabled = false # Default Product = '' # Default Tenant = '' # Default +NumericTenantID = '' # Default Environment = '' # Default Zone = '' # Default NodeID = '' # Default ``` Metering configures durable resource metering emission and the coarse deployment/node identity dimensions stamped on emitted MeterRecords and -MeterSnapshots. These are plumbed to every LOOP plugin via its environment. +MeterSnapshots. These are plumbed to LOOP plugins via env. ### MeterRecordsEnabled ```toml @@ -2604,7 +2605,13 @@ Product is the deployment product identity dimension, e.g. 'cre'. ```toml Tenant = '' # Default ``` -Tenant is the deployment tenant identity dimension, e.g. 'mainline'. +Tenant is the human-readable tenant name, e.g. 'mainline'. + +### NumericTenantID +```toml +NumericTenantID = '' # Default +``` +NumericTenantID is the numbered tenant identifier represented as a string. ### Environment ```toml diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index 158b8a46642..149c74a4baa 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -175,11 +175,12 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { if m.cfgMetering != nil { envCfg.MeterRecordsEnabled = m.cfgMetering.MeterRecordsEnabled() envCfg.MeterSnapshotsEnabled = m.cfgMetering.MeterSnapshotsEnabled() - envCfg.MeteringProduct = m.cfgMetering.Product() - envCfg.MeteringTenant = m.cfgMetering.Tenant() - envCfg.MeteringEnvironment = m.cfgMetering.Environment() - envCfg.MeteringZone = m.cfgMetering.Zone() - envCfg.MeteringNodeID = m.cfgMetering.NodeID() + envCfg.MeterProduct = m.cfgMetering.Product() + envCfg.MeterTenant = m.cfgMetering.Tenant() + envCfg.MeterNumericTenantID = m.cfgMetering.NumericTenantID() + envCfg.MeterEnvironment = m.cfgMetering.Environment() + envCfg.MeterZone = m.cfgMetering.Zone() + envCfg.MeterNodeID = m.cfgMetering.NodeID() } m.lggr.Debugf("Registered loopp %q with port %d", id, envCfg.PrometheusPort) diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index 80216f0e7f5..ede01b4ed87 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -98,6 +98,7 @@ func (m mockCfgMetering) MeterRecordsEnabled() bool { return true } func (m mockCfgMetering) MeterSnapshotsEnabled() bool { return true } func (m mockCfgMetering) Product() string { return "cre" } func (m mockCfgMetering) Tenant() string { return "mainline" } +func (m mockCfgMetering) NumericTenantID() string { return "42" } func (m mockCfgMetering) Environment() string { return "production" } func (m mockCfgMetering) Zone() string { return "wf-zone-a" } func (m mockCfgMetering) NodeID() string { return "csa-pubkey-1" } @@ -264,11 +265,12 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, 2048, envCfg.TelemetryLogMaxQueueSize) require.True(t, envCfg.MeterRecordsEnabled) require.True(t, envCfg.MeterSnapshotsEnabled) - require.Equal(t, "cre", envCfg.MeteringProduct) - require.Equal(t, "mainline", envCfg.MeteringTenant) - require.Equal(t, "production", envCfg.MeteringEnvironment) - require.Equal(t, "wf-zone-a", envCfg.MeteringZone) - require.Equal(t, "csa-pubkey-1", envCfg.MeteringNodeID) + require.Equal(t, "cre", envCfg.MeterProduct) + require.Equal(t, "mainline", envCfg.MeterTenant) + require.Equal(t, "42", envCfg.MeterNumericTenantID) + require.Equal(t, "production", envCfg.MeterEnvironment) + require.Equal(t, "wf-zone-a", envCfg.MeterZone) + require.Equal(t, "csa-pubkey-1", envCfg.MeterNodeID) require.Equal(t, "example.com/chip-ingress", envCfg.ChipIngressEndpoint) require.False(t, envCfg.ChipIngressBatchEmitterEnabled) From d288672276de9bda21ff1fb4d2c4988ad5537250 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 9 Jul 2026 14:17:39 -0400 Subject: [PATCH 05/14] feat(metering): TOML-only gating, node_id logical name, syncer delta re-anchor Phase 4 of the delta-based metering redesign, built against the new chainlink-common resourcemanager API (EmitDelta/EmitUsage, RM-generated event_id, no UtilizationFields.EventID). TOML-only gating (kills split-brain): - Delete env.MeterRecordsEnabled and the CL_METER_RECORDS_ENABLED LOOP pass-through (provably shadowed by AsCmdEnv) + its subtests. - Delete cre.meterRecordsEnabled(); gate the syncer ResourceManager on cfg.Metering().MeterRecordsEnabled()/MeterSnapshotsEnabled(). - Default [Metering].Product = "cre" (accessor + core.toml + CONFIG.md). Dead-plumbing removal: - Remove standardcapabilities.NodeIdentity struct, delegate field/ctor param, and its population in application.go. Identity delivery is loop.EnvConfig. node_id logical name: - loop_registry already sources NodeID from [Metering].NodeID; fix fixtures (csa-pubkey-1 -> clp-cre-wf-zone-a-1) and drop the CSA-pubkey comment. Host-side CapDONID (rule 8): - Delegate already resolves and populates CapabilityDonID on the dependencies handed to capability LOOPs at Initialise; add a test asserting a nonzero CapDONID round-trips (and zero is preserved for the workflow-DON fallback). Syncer v2 metering re-anchor (handler.go): - Emit anchors on artifact persistence: EmitDelta(+1) when a new spec's artifacts are first persisted, EmitDelta(-1) when a real delete removes them. Pause/activate and status-only updates emit nothing. - Delete emitGracefulCloseReleases and its call in close (no lifecycle emits). - GetUtilization enumerates persisted specs (WorkflowSpecsDS.GetWorkflowSpecList) at value 1 per workflow; org resolved from the stored owner via the shared CachingOrgResolver. Records resolve org from payload owner via ResolveOrEmpty. - Default ResourceManager to nil (nil-guarded), reuse one emitSpecDelta helper. Regenerate txtar validate/merge goldens and effective TOMLs for the new Product default and the Tenant/NumericTenantID dimensions. Note: local replace directives point chainlink, core/scripts and deployment at the ../chainlink-common and ../chainlink-protos/metering/go siblings so this branch builds against the finished upstream API; the maintainer drops these and bumps the module pins at merge time. --- core/config/docs/core.toml | 10 +- core/config/env/env.go | 4 - core/scripts/go.mod | 9 +- core/scripts/go.sum | 2 + core/services/chainlink/application.go | 10 -- core/services/chainlink/config_metering.go | 5 +- .../chainlink/config_metering_test.go | 9 +- core/services/chainlink/config_test.go | 2 +- .../testdata/config-empty-effective.toml | 2 +- .../chainlink/testdata/config-full.toml | 2 +- .../config-multi-chain-effective.toml | 2 +- core/services/cre/cre.go | 41 ++--- .../services/standardcapabilities/delegate.go | 17 -- .../standard_capabilities.go | 10 +- .../standard_capabilities_test.go | 69 ++++---- core/services/workflows/artifacts/v2/orm.go | 21 +++ core/services/workflows/artifacts/v2/store.go | 6 + core/services/workflows/syncer/v2/handler.go | 164 +++++++++--------- .../syncer/v2/handler_metering_test.go | 154 +++++++--------- .../workflows/syncer/v2/handler_test.go | 13 ++ .../services/workflows/syncer/v2/mocks/orm.go | 58 +++++++ .../testdata/config-empty-effective.toml | 2 +- core/web/resolver/testdata/config-full.toml | 2 +- .../config-multi-chain-effective.toml | 2 +- deployment/go.mod | 9 +- deployment/go.sum | 2 + docs/CONFIG.md | 12 +- go.mod | 11 +- go.sum | 2 + plugins/loop_registry_test.go | 4 +- .../scripts/config/merge_raw_configs.txtar | 4 +- testdata/scripts/node/validate/default.txtar | 4 +- .../node/validate/defaults-override.txtar | 4 +- .../disk-based-logging-disabled.txtar | 4 +- .../validate/disk-based-logging-no-dir.txtar | 4 +- .../node/validate/disk-based-logging.txtar | 4 +- .../node/validate/fallback-override.txtar | 4 +- .../node/validate/invalid-ocr-p2p.txtar | 4 +- testdata/scripts/node/validate/invalid.txtar | 4 +- testdata/scripts/node/validate/valid.txtar | 4 +- testdata/scripts/node/validate/warnings.txtar | 4 +- 41 files changed, 403 insertions(+), 297 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index f4e389b92b9..c5b29cac3a6 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -924,7 +924,13 @@ Prefixes = ["go_"] # Default # Metering configures durable resource metering emission and the coarse # deployment/node identity dimensions stamped on emitted MeterRecords and -# MeterSnapshots. These are plumbed to LOOP plugins via env. +# MeterSnapshots. This TOML section is the single authority for metering on the +# core node; there is no environment-variable gate. For capability LOOP plugins +# the values are forwarded unchanged over the plugin environment +# (loop.EnvConfig), which is only a child-process transport produced from this +# config, not a separate gate. Snapshots are emitted on a fixed internal +# interval and are bucket-aligned (each snapshot timestamp is truncated to the +# interval) so cross-node snapshot buckets agree. [Metering] # MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. MeterRecordsEnabled = false # Default @@ -932,7 +938,7 @@ MeterRecordsEnabled = false # Default # Requires MeterRecordsEnabled = true. MeterSnapshotsEnabled = false # Default # Product is the deployment product identity dimension, e.g. 'cre'. -Product = '' # Default +Product = 'cre' # Default # Tenant is the human-readable tenant name, e.g. 'mainline'. Tenant = '' # Default # NumericTenantID is the numbered tenant identifier represented as a string. diff --git a/core/config/env/env.go b/core/config/env/env.go index eed36c80152..dc5b4951fbc 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -14,10 +14,6 @@ var ( DatabaseAllowSimplePasswords = Var("CL_DATABASE_ALLOW_SIMPLE_PASSWORDS") IgnorePrereleaseVersionCheck = Var("CL_IGNORE_PRE_RELEASE_VERSION_CHECK") SkipAppVersionCheck = Var("CL_SKIP_APP_VERSION_CHECK") - // MeterRecordsEnabled gates emission of metering.v1.MeterRecord events for - // durable CRE resources. Accepts strconv.ParseBool values; default false. - // Temporary deploy gate; promotion to TOML config is tracked by SHARED-2718. - MeterRecordsEnabled = Var("CL_METER_RECORDS_ENABLED") DatabaseURL = Secret("CL_DATABASE_URL") DatabaseBackupURL = Secret("CL_DATABASE_BACKUP_URL") diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 7352bd2ca4b..493f97c777d 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -5,6 +5,13 @@ go 1.26.4 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ +// Local development replaces for the delta-based metering redesign, kept +// consistent with the root module. The maintainer drops these and bumps the +// module pins at merge time. +replace github.com/smartcontractkit/chainlink-common => ../../../chainlink-common + +replace github.com/smartcontractkit/chainlink-protos/metering/go => ../../../chainlink-protos/metering/go + replace github.com/smartcontractkit/chainlink/deployment => ../../deployment replace github.com/smartcontractkit/chainlink/system-tests/lib => ../../system-tests/lib @@ -508,7 +515,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index afa95889506..1167d62f457 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1630,6 +1630,8 @@ github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-8 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index fe5da2b1893..19a754b642e 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -676,16 +676,6 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err atomicSettings, creServices.OCRConfigService, cfg.Capabilities().Local(), - // Host-injected deployment/node metering identity for spawned capability - // LOOPs. Sourced once here from node config: product constant, env/zone - // from [Telemetry.ResourceAttributes], node_id = the same CSA pubkey hex - // the node uses for beholder auth and the engine's node_id. - standardcapabilities.NodeIdentity{ - Product: "cre", - Environment: cfg.Telemetry().ResourceAttributes()["env"], - Zone: cfg.Telemetry().ResourceAttributes()["zone"], - NodeID: csaPubKeyHex, - }, ) delegates[job.StandardCapabilities] = stdcapDelegate if creServices.SetDelegatesDeps != nil { diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go index fa0c8f36843..ac8bd147887 100644 --- a/core/services/chainlink/config_metering.go +++ b/core/services/chainlink/config_metering.go @@ -22,9 +22,12 @@ func (b *meteringConfig) MeterSnapshotsEnabled() bool { return *b.s.MeterSnapshotsEnabled } +// Product defaults to "cre" (resourcemanager.DefaultMeteringProduct) when unset, +// so the syncer's metering identity matches the capability plugins' fallback and +// metering can never be enabled with an empty product dimension. func (b *meteringConfig) Product() string { if b.s.Product == nil { - return "" + return "cre" } return *b.s.Product } diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go index 56d61af1925..093cb806c66 100644 --- a/core/services/chainlink/config_metering_test.go +++ b/core/services/chainlink/config_metering_test.go @@ -13,7 +13,10 @@ func TestMeteringConfig(t *testing.T) { mc := meteringConfig{s: toml.Metering{}} assert.False(t, mc.MeterRecordsEnabled()) assert.False(t, mc.MeterSnapshotsEnabled()) - assert.Empty(t, mc.Product()) + // Product defaults to "cre" so metering can never be enabled with an + // empty product dimension and the syncer identity matches the plugins' + // fallback. + assert.Equal(t, "cre", mc.Product()) assert.Empty(t, mc.Tenant()) assert.Empty(t, mc.NumericTenantID()) assert.Empty(t, mc.Environment()) @@ -30,7 +33,7 @@ func TestMeteringConfig(t *testing.T) { NumericTenantID: ptr("42"), Environment: ptr("production"), Zone: ptr("wf-zone-a"), - NodeID: ptr("csa-pubkey-1"), + NodeID: ptr("clp-cre-wf-zone-a-1"), }} assert.True(t, mc.MeterRecordsEnabled()) assert.True(t, mc.MeterSnapshotsEnabled()) @@ -39,6 +42,6 @@ func TestMeteringConfig(t *testing.T) { assert.Equal(t, "42", mc.NumericTenantID()) assert.Equal(t, "production", mc.Environment()) assert.Equal(t, "wf-zone-a", mc.Zone()) - assert.Equal(t, "csa-pubkey-1", mc.NodeID()) + assert.Equal(t, "clp-cre-wf-zone-a-1", mc.NodeID()) }) } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index a5e654065e9..de4d3775755 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -586,7 +586,7 @@ func TestConfig_Marshal(t *testing.T) { NumericTenantID: ptr("42"), Environment: ptr("production"), Zone: ptr("wf-zone-a"), - NodeID: ptr("csa-pubkey-1"), + NodeID: ptr("clp-cre-wf-zone-a-1"), } full.CRE = toml.CreConfig{ UseLocalTimeProvider: ptr(true), diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index cb9ed7589f7..31673d9262d 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -367,7 +367,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 907bb21bc98..3514c36f3ff 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -411,7 +411,7 @@ Tenant = 'mainline' NumericTenantID = '42' Environment = 'production' Zone = 'wf-zone-a' -NodeID = 'csa-pubkey-1' +NodeID = 'clp-cre-wf-zone-a-1' [Workflows] [Workflows.Limits] diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index d42d768567e..e56f5660440 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -367,7 +367,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index e3da8377357..df8c192180c 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -45,7 +45,6 @@ import ( remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" capStreams "github.com/smartcontractkit/chainlink/v2/core/capabilities/streams" "github.com/smartcontractkit/chainlink/v2/core/config" - "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/ocr/capregconfig" @@ -245,8 +244,16 @@ func (s *Services) newSubservices( return nil, fmt.Errorf("could not create org resolver: %w", ierr) } fallbackResolver := orgresolver.NewOrgResolverWithFallback(inner, lggr) - s.OrgResolver = fallbackResolver - srvs = append(srvs, fallbackResolver) + // Wrap in a caching resolver so the metering snapshot path + // (Meterable.GetUtilization, which is contractually no-network) can + // resolve org attribution from memory. This single instance is shared: + // the syncer's record path resolves through it too, warming the cache + // for snapshots. The caching resolver owns the inner resolver's + // lifecycle (its Start/Close drive the inner), so only the caching + // resolver is added to the service list. + cachingResolver := orgresolver.NewCaching(fallbackResolver, resourcemanager.DefaultSnapshotInterval) + s.OrgResolver = cachingResolver + srvs = append(srvs, cachingResolver) } else { lggr.Warn("Skipping orgResolver, no linking service configured") } @@ -752,24 +759,6 @@ func newSyncerMeterIdentity(cfg Config) resourcemanager.ResourceIdentity { return id } -// meterRecordsEnabled reads the CL_METER_RECORDS_ENABLED env var gating emission of -// metering.v1.MeterRecord events. Unset, empty, or invalid values disable emission. -// Promotion of this gate to TOML config is tracked by SHARED-2718. -func meterRecordsEnabled(lggr logger.Logger) bool { - v := env.MeterRecordsEnabled.Get() - if v == "" { - return false - } - enabled, err := strconv.ParseBool(v) - if err != nil { - // Warn, not error, matching the capability producers: a bad gate value - // disables emission but must never disturb the node. - lggr.Warnw("Invalid CL_METER_RECORDS_ENABLED value; meter record emission disabled", "value", v, "err", err) - return false - } - return enabled -} - func newShardOrchestratorClient(cfg Config, lggr logger.Logger) (*shardorchestrator.Client, error) { shardID := cfg.Sharding().ShardIndex() if shardID == 0 { @@ -1061,7 +1050,11 @@ func newWorkflowRegistrySyncerV2( } shardRoutingSteady = shardownership.NewSteadySignal(shardownership.WithSteadySignalMetrics(steadyMetrics)) } - meterRecords := meterRecordsEnabled(lggr) + // TOML [Metering] is the single authority for the emission gate; the + // deleted CL_METER_RECORDS_ENABLED node env var no longer participates. + meteringCfg := cfg.Metering() + meterRecordsEnabled := meteringCfg != nil && meteringCfg.MeterRecordsEnabled() + meterSnapshotsEnabled := meteringCfg != nil && meteringCfg.MeterSnapshotsEnabled() handlerOpts := []syncerV2.EventHandlerOption{ syncerV2.WithBillingClient(billingClient), @@ -1076,8 +1069,8 @@ func newWorkflowRegistrySyncerV2( // SnapshotInterval enables the periodic absolute-state snapshot loop; the // RM is otherwise a no-op when metering is disabled. syncerV2.WithResourceManager(resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ - MeterRecordsEnabled: meterRecords, - MeterSnapshotsEnabled: meterRecords, + MeterRecordsEnabled: meterRecordsEnabled, + MeterSnapshotsEnabled: meterSnapshotsEnabled, Emitter: beholder.GetEmitter(), SnapshotInterval: resourcemanager.DefaultSnapshotInterval, })), diff --git a/core/services/standardcapabilities/delegate.go b/core/services/standardcapabilities/delegate.go index 7282ff6e1cf..63a37c1bcef 100644 --- a/core/services/standardcapabilities/delegate.go +++ b/core/services/standardcapabilities/delegate.go @@ -46,20 +46,6 @@ type RelayGetter interface { GetIDToRelayerMap() map[types.RelayID]loop.Relayer } -// NodeIdentity is the host-injected deployment/node metering identity sourced at -// node startup. It is retained on the delegate for compatibility with launcher -// wiring while metering identity delivery is handled through loop.EnvConfig. -type NodeIdentity struct { - // Product is the deployment product, e.g. "cre". - Product string - // Environment is the deployment environment, from [Telemetry.ResourceAttributes]["env"]. - Environment string - // Zone is the deployment zone, from [Telemetry.ResourceAttributes]["zone"]. - Zone string - // NodeID is the node's logical name (not the CSA public key). - NodeID string -} - type Delegate struct { logger logger.Logger ds sqlutil.DataSource @@ -80,7 +66,6 @@ type Delegate struct { creSettings core.SettingsBroadcaster ocrConfigService capregconfig.OCRConfigService localCfg coreconfig.LocalCapabilities - nodeIdentity NodeIdentity initErr error isNewlyCreatedJob bool @@ -113,7 +98,6 @@ func NewDelegate( creSettings core.SettingsBroadcaster, ocrConfigService capregconfig.OCRConfigService, localCfg coreconfig.LocalCapabilities, - nodeIdentity NodeIdentity, opts ...func(*gateway.RoundRobinSelector), ) *Delegate { initErr := registerOptionalMockStreamsTrigger(logger, localCfg, registry) @@ -141,7 +125,6 @@ func NewDelegate( creSettings: creSettings, ocrConfigService: ocrConfigService, localCfg: localCfg, - nodeIdentity: nodeIdentity, initErr: initErr, selectorOpts: opts, } diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index 01ea074dc44..af549670559 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -106,12 +106,10 @@ func (s *StandardCapabilities) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to parse capabilities env file: %w", err) } - // Pass through the node's meter-record emission gate so capability LOOPPs - // inherit it: plugins.NewCmdFactory builds the child env from CmdConfig.Env - // only and does not inherit os.Environ(). - if v := env.MeterRecordsEnabled.Get(); v != "" { - envVars = append(envVars, fmt.Sprintf("%s=%s", env.MeterRecordsEnabled, v)) - } + // Metering identity and the emission gate reach capability LOOPs through + // loop.EnvConfig (produced from the node's TOML [Metering] via + // EnvConfig.AsCmdEnv), so there is no meter-record env var to pass + // through here. cmdFn, opts, err := s.pluginRegistrar.RegisterLOOP(plugins.CmdConfig{ ID: s.log.Name(), Cmd: s.command, diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index 3884c702d3c..d24096b6fac 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -74,41 +74,14 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { t.Run("env file unset results in empty CmdConfig.Env", func(t *testing.T) { t.Setenv(string(env.CapabilitiesPlugin.Env), "") - t.Setenv(string(env.MeterRecordsEnabled), "") cfg, err := startAndCapture(t) require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") require.Contains(t, err.Error(), capturingRegistrarErr) require.Empty(t, cfg.Env, - "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset") - }) - - t.Run("CL_METER_RECORDS_ENABLED set on the node is passed through to the LOOPP", func(t *testing.T) { - envFile := writeEnvFile(t, "FOO=bar\n") - t.Setenv(string(env.CapabilitiesPlugin.Env), envFile) - t.Setenv(string(env.MeterRecordsEnabled), "true") - - cfg, err := startAndCapture(t) - require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") - require.Contains(t, err.Error(), capturingRegistrarErr) - - require.Contains(t, cfg.Env, "CL_METER_RECORDS_ENABLED=true", - "the node's meter-record emission gate must reach capability LOOPPs, which do not inherit os.Environ()") - require.Contains(t, cfg.Env, "FOO=bar", - "the pass-through must not displace entries from the operator-supplied env file") - }) - - t.Run("CL_METER_RECORDS_ENABLED unset is not forwarded", func(t *testing.T) { - t.Setenv(string(env.CapabilitiesPlugin.Env), "") - t.Setenv(string(env.MeterRecordsEnabled), "") - - cfg, err := startAndCapture(t) - require.Error(t, err, "expected synthetic Start failure from capturingRegistrar") - require.Contains(t, err.Error(), capturingRegistrarErr) - - require.Empty(t, cfg.Env, - "no CL_METER_RECORDS_ENABLED entry should be forwarded when the gate is unset on the node") + "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset; "+ + "metering config reaches LOOPs via loop.EnvConfig, not this pass-through") }) t.Run("missing env file fails Start before RegisterLOOP", func(t *testing.T) { @@ -152,6 +125,44 @@ func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { require.Equal(t, want.Config, got.Config, "config should be re-delivered to LOOP Initialise dependencies") } +// TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP asserts that the +// host-resolved capability DON ID is carried on the dependencies delivered to +// the capability LOOP at Initialise. Without this, a trigger producer silently +// falls back to the consumer workflow's DON for metering identity and event +// labels even when the node knows which DON it is serving. +func TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP(t *testing.T) { + t.Run("nonzero DON ID round-trips when the DON is known", func(t *testing.T) { + const knownDonID = uint32(42) + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{CapabilityDonID: knownDonID}, + ) + + got := std.initialiseDependencies() + + require.Equal(t, knownDonID, got.CapabilityDonID, + "the host-resolved capability DON ID must reach the LOOP at Initialise") + }) + + t.Run("zero DON ID is preserved when the host could not resolve one", func(t *testing.T) { + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{}, + ) + + got := std.initialiseDependencies() + + require.Zero(t, got.CapabilityDonID, + "an unresolved DON ID stays zero so the LOOP can fall back to the workflow DON") + }) +} + const capturingRegistrarErr = "capturingRegistrar: stop after capture" // capturingRegistrar records the CmdConfig handed to RegisterLOOP and then diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 31395bce8f3..a9467a3cd59 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -20,6 +20,11 @@ type WorkflowSpecsDS interface { // GetWorkflowSpecByID returns the workflow spec for the given workflowID. GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) + // GetWorkflowSpecList returns the persisted workflow specs. It projects only + // the identity columns needed by the metering snapshot path (workflow_id and + // workflow_owner); other fields are left zero. + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) + // DeleteWorkflowSpec deletes the workflow spec for the given workflow ID. DeleteWorkflowSpec(ctx context.Context, id string) error @@ -129,6 +134,22 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp return &spec, nil } +// GetWorkflowSpecList returns all persisted workflow specs, projecting only the +// identity columns the metering snapshot path needs (workflow_id, workflow_owner). +func (orm *orm) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + query := ` + SELECT workflow_id, workflow_owner + FROM workflow_specs_v2 + ` + + var specs []*job.WorkflowSpec + if err := orm.ds.SelectContext(ctx, &specs, query); err != nil { + return nil, err + } + + return specs, nil +} + func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) error { query := ` DELETE FROM workflow_specs_v2 diff --git a/core/services/workflows/artifacts/v2/store.go b/core/services/workflows/artifacts/v2/store.go index 0eeb694cc6a..deb27c77632 100644 --- a/core/services/workflows/artifacts/v2/store.go +++ b/core/services/workflows/artifacts/v2/store.go @@ -234,6 +234,12 @@ func (h *Store) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.Wo return spec, err } +// GetWorkflowSpecList returns the persisted workflow specs (identity columns +// only). It backs the metering snapshot path. +func (h *Store) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.orm.GetWorkflowSpecList(ctx) +} + func (h *Store) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { return h.orm.UpsertWorkflowSpec(ctx, spec) } diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index ea7e27d8d89..5f47c07fb95 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -308,6 +308,10 @@ func WithModuleEngineVersion(v string) func(*eventHandler) { type WorkflowArtifactsStore interface { FetchWorkflowArtifacts(ctx context.Context, workflowID, binaryIdentifier, configIdentifier string) ([]byte, []byte, error) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + // GetWorkflowSpecList returns the persisted workflow specs. It backs the + // metering snapshot path (Meterable.GetUtilization), which enumerates the + // durable specs rather than running engines. + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error @@ -361,7 +365,9 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - resourceManager: resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{}), // default to disabled, enable via WithResourceManager + // resourceManager defaults to nil (metering off); it is set only via + // WithResourceManager. A nil manager keeps every emission site guarded + // and avoids registering duplicate OTel metering instruments. // Default identity carries only the service-level constants; the coarse // deployment/node/DON dimensions are filled in by WithIdentity in cre.go. meterIdentity: resourcemanager.ResourceIdentity{ @@ -441,10 +447,10 @@ func (h *eventHandler) close() error { h.moduleLRU.Close() } es := h.engineRegistry.PopAll() - // Emit a graceful-close RELEASE for each still-running engine before popping - // them, so a clean shutdown pairs every snapshot RESERVE. Best-effort and - // fail-open: metering must never gate shutdown. - h.emitGracefulCloseReleases(context.Background(), es) + // No metering is emitted on close: meter records anchor on workflow-spec + // storage transitions, not engine lifecycle, so stopping an engine at + // shutdown leaves the persisted spec (and therefore its metered level) + // untouched. A spec that is genuinely released stops being snapshotted. // Stop snapshotting this handler, then close the ResourceManager service. if h.rmUnregister != nil { h.rmUnregister() @@ -675,7 +681,9 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, originatingEvent, payload.WorkflowID.Hex()) + // A new spec's artifacts were persisted for the first time: the durable + // workflow_specs_v2 resource gained one unit, so emit a +1 delta. + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner)) spec = newSpec case spec.WorkflowID != payload.WorkflowID.Hex(): @@ -683,7 +691,9 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RESERVE, originatingEvent, payload.WorkflowID.Hex()) + // A different spec's artifacts were persisted under this key: the newly + // stored spec is a fresh durable resource, so emit a +1 delta. + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner)) spec = newSpec case spec.Status != status: @@ -691,7 +701,9 @@ func (h *eventHandler) workflowRegisteredEvent( if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { return fmt.Errorf("failed to update workflow spec: %w", innerErr) } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_UPDATE, originatingEvent, payload.WorkflowID.Hex()) + // Status-only update (e.g. activate an already-stored spec): the spec + // stays stored, so there is no artifact-persistence transition and no + // meter delta. The level is unchanged and remains covered by snapshots. } // Next, let's synchronize the engine. @@ -959,6 +971,16 @@ func (h *eventHandler) workflowDeletedEvent( // At the same time, popping the engine should occur last to allow deletes to be retried if any of the // prior steps fail. workflowID := payload.WorkflowID.Hex() + // Capture the stored owner before deletion so a real-delete -1 delta can + // resolve org attribution from durable state. Only needed for actual + // deletions (pause emits no delta); fail-open, an empty owner yields an + // empty org. + var deleteOwner string + if originatingEvent == WorkflowDeleted { + if existing, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, workflowID); gerr == nil && existing != nil { + deleteOwner = existing.WorkflowOwner + } + } e, ok := h.engineRegistry.Get(payload.WorkflowID) var drainable DrainableService var isDrainable bool @@ -990,7 +1012,13 @@ func (h *eventHandler) workflowDeletedEvent( if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } - h.emitMeterRecord(ctx, meteringpb.MeterAction_METER_ACTION_RELEASE, originatingEvent, workflowID) + // A real delete removed the persisted spec, so emit a -1 delta. Pause is + // delegated here too (originatingEvent == WorkflowPaused); its release is + // realized by the spec's absence from subsequent snapshots, so pause emits + // no delta. + if originatingEvent == WorkflowDeleted { + h.emitSpecDelta(ctx, -1, workflowID, deleteOwner) + } h.cleanupModuleCache(payload.WorkflowID.Hex()) @@ -1011,35 +1039,30 @@ func (h *eventHandler) workflowDeletedEvent( return nil } -// emitMeterRecord emits a metering.v1.MeterRecord for a workflow_specs_v2 mutation. -// Callers must invoke it only after the database mutation has succeeded. Emission is -// fail-open: it never returns an error and must never affect event handling. -// Consumer-side dedup can use the emitted identity + action + -// utilization(resource_id/event_id). +// emitSpecDelta emits one metering.v1.MeterRecord (METER_ACTION_UPDATE) +// capturing a signed ±delta to the durable workflow_specs_v2 level for +// workflowID. Each record is a first-class billing event: a +1 when a new +// spec's artifacts are persisted, a -1 when a spec is deleted. It is the single +// emission helper for every syncer meter record; callers invoke it only after +// the workflow-spec storage mutation has committed. // -// Known lost-record windows (consumer-side reconciliation tracked by SHARED-2141): -// - A node crash between the workflow spec database write and this emit loses the -// record (e.g. the RESERVE for a new spec): the syncer has no recovery path that -// re-emits records for mutations persisted before the crash. -// - A workflow paused or deleted while the node is down loses its RELEASE: -// reconciliation only synthesizes events for engine-registry entries, and the -// restarted node has no engine registered for the already-removed workflow. -func (h *eventHandler) emitMeterRecord(ctx context.Context, action meteringpb.MeterAction, originatingEvent WorkflowRegistryEventName, workflowID string) { +// owner is the workflow owner as stored in durable state; the organization is +// resolved fail-open at emit time via ResolveOrEmpty and is never persisted. +// Emission is fail-open and returns no error, so metering can never gate, +// delay, or fail the storage operation it records. event_id is generated per +// emission by the ResourceManager (a fresh UUIDv4) and is the consumer's dedup +// key. +func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowID, owner string) { if h.resourceManager == nil { return } - // resource_id = workflow_id (the syncer has no shared physical resource); - // event_id = originating event name so lifecycle edges remain distinguishable. - identity := h.baseIdentity() - utilizations := []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meterResourceType, - ResourceID: workflowID, - EventID: string(originatingEvent), - }), - } - h.resourceManager.EmitMeterRecord(ctx, identity, action, - utilizations) + orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) + // resource_id = workflow_id (the syncer meters one durable spec per workflow). + h.resourceManager.EmitDelta(ctx, h.baseIdentity(), delta, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }) } // baseIdentity returns the handler's metering identity with the workflow DON id @@ -1070,24 +1093,38 @@ func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { } // GetUtilization implements resourcemanager.Meterable: it returns one -// SnapshotEntry per running engine, each with one Utilization at value 1 -// (each running workflow holds one workflow_specs_v2 reservation). It is a pure -// in-memory read over the engine registry — each entry's resource_id is the -// workflow_id, which fully identifies the resource (Design 1; reconciling -// persisted-but-not-running specs is the Design 2 follow-up, see -// snapshotDesign2Followup below). -func (h *eventHandler) GetUtilization(_ context.Context) []resourcemanager.SnapshotEntry { +// SnapshotEntry per persisted workflow_specs_v2 spec, each at level 1, so the +// periodic absolute-state snapshot agrees with the +1/-1 deltas emitted on +// artifact-persistence transitions. The durable resource is the stored spec, +// not the running engine, so this enumerates persisted specs (via the store's +// list) rather than the engine registry — that way paused-but-stored specs and +// specs whose engine failed to start are still accounted for, and a delete is +// reflected by the spec's absence from the next snapshot. +// +// resource_id is the workflow_id; the organization is resolved from the spec's +// stored owner through the caching resolver, which serves from memory so this +// tick stays effectively no-network. Fail-open: a store error yields no entries +// for this tick. +func (h *eventHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + specs, err := h.workflowArtifactsStore.GetWorkflowSpecList(ctx) + if err != nil { + h.lggr.Warnw("failed to list persisted workflow specs for metering snapshot; skipping tick", "err", err) + return nil + } base := h.baseIdentity() - engines := h.engineRegistry.GetAll() - entries := make([]resourcemanager.SnapshotEntry, 0, len(engines)) - for _, e := range engines { - workflowID := e.WorkflowID.Hex() + entries := make([]resourcemanager.SnapshotEntry, 0, len(specs)) + for _, spec := range specs { + if spec == nil { + continue + } + orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, spec.WorkflowOwner, h.lggr) entries = append(entries, resourcemanager.SnapshotEntry{ Identity: base, Utilizations: []*meteringpb.Utilization{ resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, - ResourceID: workflowID, + ResourceID: spec.WorkflowID, + OrgID: orgID, }), }, }) @@ -1095,41 +1132,6 @@ func (h *eventHandler) GetUtilization(_ context.Context) []resourcemanager.Snaps return entries } -// snapshotDesign2Followup documents the deferred reconciliation work. Design 1 -// (implemented here) snapshots only engines that are currently running in this -// node's engine registry. Design 2 — the follow-up — would additionally -// enumerate persisted-but-not-running specs via WorkflowSpecsDS.ListAll so that -// snapshots also account for specs the database has but for which no engine is -// live (e.g. paused specs, or specs whose engine failed to start). That requires -// a store-backed read on the tick (not a cheap in-memory snapshot) and a policy -// for the utilization value of a non-running spec, so it is intentionally left -// out of the in-memory GetUtilization above. Tracked as the syncer Design 2 -// follow-up. -const snapshotDesign2Followup = "Design 2: reconcile persisted-but-not-running specs via WorkflowSpecsDS.ListAll" - -// emitGracefulCloseReleases emits a RELEASE meter record for every engine still -// in the registry at shutdown, so a clean close pairs every snapshot RESERVE. It -// is fail-open like all metering emission. Called from close before the engines -// are popped. -func (h *eventHandler) emitGracefulCloseReleases(ctx context.Context, engines []ServiceWithMetadata) { - if h.resourceManager == nil { - return - } - identity := h.baseIdentity() - for _, e := range engines { - workflowID := e.WorkflowID.Hex() - utilizations := []*meteringpb.Utilization{ - resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ - ResourceType: meterResourceType, - ResourceID: workflowID, - EventID: string(WorkflowDeleted), - }), - } - h.resourceManager.EmitMeterRecord(ctx, identity, meteringpb.MeterAction_METER_ACTION_RELEASE, - utilizations) - } -} - // tryEngineCleanup attempts to stop the workflow engine for the given workflow ID. Does nothing if the // workflow engine is not running. func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index 9f31cc5b03b..90d6789e798 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -2,13 +2,13 @@ package v2 import ( "context" - "encoding/hex" "errors" "math/big" "sync" "testing" "time" + "github.com/google/uuid" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,10 +20,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" - pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" "github.com/smartcontractkit/chainlink/v2/core/capabilities" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" @@ -117,6 +117,7 @@ func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, nil, true, registry, + &confidentialrelay.ExecutionHandlers{}, NewEngineRegistry(), custmsg.NewLabeler(), limiters, @@ -133,29 +134,36 @@ func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, return h } -func requireMeterRecord(t *testing.T, record *meteringpb.MeterRecord, action meteringpb.MeterAction, originatingEvent WorkflowRegistryEventName, workflowID string) { +// requireSpecDelta asserts that record is a workflow-syncer-v2 spec delta: a +// METER_ACTION_UPDATE carrying a single utilization with the given signed value +// and resource_id (= workflow_id). event_id must be a manager-generated UUID, +// not the originating event name. +func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workflowID string) { t.Helper() require.NotNil(t, record.Identity) assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) assert.Equal(t, "workflow_specs_v2", record.Identity.ResourcePool) - assert.Equal(t, action, record.Action) + // Every syncer meter record is a signed level delta (UPDATE), never a + // RESERVE/RELEASE lifecycle edge. + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.Action) assert.NotNil(t, record.Timestamp) require.Len(t, record.Utilizations, 1) util := record.Utilizations[0] - assert.Equal(t, "1", util.Value) + assert.Equal(t, value, util.Value) assert.Equal(t, "operations", util.ResourceType) // resource_id = workflow_id for the syncer (no shared physical resource). assert.Equal(t, workflowID, util.ResourceId) - assert.Equal(t, string(originatingEvent), util.EventId) + // event_id is generated per emission by the ResourceManager (a UUIDv4). + _, parseErr := uuid.Parse(util.EventId) + assert.NoError(t, parseErr, "event_id must be a manager-generated UUID, got %q", util.EventId) } func Test_meterRecords(t *testing.T) { t.Parallel() wfOwner := []byte{0xaa, 0xbb, 0xcc, 0xdd} - wfOwnerHex := hex.EncodeToString(wfOwner) - t.Run("registered event creating a new spec emits RESERVE", func(t *testing.T) { + t.Run("registered event persisting a new spec emits +1", func(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) @@ -171,10 +179,10 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RESERVE, WorkflowRegistered, wfID.Hex()) + requireSpecDelta(t, records[0], "1", wfID.Hex()) }) - t.Run("retried registered event emits equivalent utilization identity", func(t *testing.T) { + t.Run("retried registered event emits a fresh event_id per emission", func(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} // The stub never returns a stored spec, so each call replays the @@ -198,70 +206,63 @@ func Test_meterRecords(t *testing.T) { assert.Equal(t, records[0].Identity.GetService(), records[1].Identity.GetService()) assert.Equal(t, records[0].Identity.GetResourcePool(), records[1].Identity.GetResourcePool()) assert.Equal(t, records[0].Utilizations[0].GetResourceId(), records[1].Utilizations[0].GetResourceId()) - assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) + // event_id is generated fresh per emission; retries do not reuse it. + assert.NotEqual(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) }) - t.Run("activated event with existing spec emits UPDATE", func(t *testing.T) { + t.Run("activating an already-stored spec emits nothing (status-only update)", func(t *testing.T) { t.Parallel() - binary := []byte("binary-data") - config := []byte("") - giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "wf-name", binary, config, "") - require.NoError(t, err) - wfID := types.WorkflowID(giveWFID) - + wfID := types.WorkflowID{3} emitter := &recordingEmitter{} h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ spec: &job.WorkflowSpec{ - Workflow: hex.EncodeToString(binary), - Config: string(config), WorkflowID: wfID.Hex(), - Status: job.WorkflowSpecStatusPaused, - WorkflowOwner: wfOwnerHex, - WorkflowName: "wf-name", + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", }, }, newMeteringResourceManager(t, true, emitter)) - err = h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent{ - Status: WorkflowStatusActive, + // Same workflow ID already stored, only the status changes: the spec + // stays stored, so no artifact-persistence transition and no delta. + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, WorkflowID: wfID, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }) + }, WorkflowRegistered) require.NoError(t, err) - records := emitter.Records() - require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_UPDATE, WorkflowActivated, wfID.Hex()) + assert.Empty(t, emitter.Records()) }) - t.Run("paused event emits RELEASE after artifacts are deleted", func(t *testing.T) { + t.Run("paused event emits nothing", func(t *testing.T) { t.Parallel() - wfID := types.WorkflowID{3} + wfID := types.WorkflowID{4} emitter := &recordingEmitter{} h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ spec: &job.WorkflowSpec{ WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, + WorkflowOwner: "aabbccdd", }, }, newMeteringResourceManager(t, true, emitter)) require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) - records := emitter.Records() - require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowPaused, wfID.Hex()) + // Pause routes through the delete path, but the spec's release is + // realized by its absence from subsequent snapshots, not a delta. + assert.Empty(t, emitter.Records()) }) - t.Run("deleted event emits RELEASE after artifacts are deleted", func(t *testing.T) { + t.Run("deleted event emits -1 after artifacts are deleted", func(t *testing.T) { t.Parallel() - wfID := types.WorkflowID{4} + wfID := types.WorkflowID{5} emitter := &recordingEmitter{} h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ spec: &job.WorkflowSpec{ WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, + WorkflowOwner: "aabbccdd", }, }, newMeteringResourceManager(t, true, emitter)) @@ -269,40 +270,17 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + requireSpecDelta(t, records[0], "-1", wfID.Hex()) }) - t.Run("no record when creating a new spec fails", func(t *testing.T) { + t.Run("no record when persisting a new spec fails", func(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{upsertErr: assert.AnError}, newMeteringResourceManager(t, true, emitter)) err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ Status: WorkflowStatusPaused, - WorkflowID: types.WorkflowID{5}, - WorkflowOwner: wfOwner, - WorkflowName: "wf-name", - }, WorkflowRegistered) - require.ErrorIs(t, err, assert.AnError) - assert.Empty(t, emitter.Records()) - }) - - t.Run("no record when a status-only update fails", func(t *testing.T) { - t.Parallel() - wfID := types.WorkflowID{6} - emitter := &recordingEmitter{} - h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ - spec: &job.WorkflowSpec{ - WorkflowID: wfID.Hex(), - Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, - }, - upsertErr: assert.AnError, - }, newMeteringResourceManager(t, true, emitter)) - - err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ - Status: WorkflowStatusPaused, - WorkflowID: wfID, + WorkflowID: types.WorkflowID{6}, WorkflowOwner: wfOwner, WorkflowName: "wf-name", }, WorkflowRegistered) @@ -318,7 +296,7 @@ func Test_meterRecords(t *testing.T) { spec: &job.WorkflowSpec{ WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, + WorkflowOwner: "aabbccdd", }, deleteErr: assert.AnError, }, newMeteringResourceManager(t, true, emitter)) @@ -336,7 +314,7 @@ func Test_meterRecords(t *testing.T) { spec: &job.WorkflowSpec{ WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, + WorkflowOwner: "aabbccdd", }, }, newMeteringResourceManager(t, true, emitter)) @@ -353,7 +331,7 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + requireSpecDelta(t, records[0], "-1", wfID.Hex()) }) t.Run("emit failure never fails event handling", func(t *testing.T) { @@ -364,7 +342,7 @@ func Test_meterRecords(t *testing.T) { spec: &job.WorkflowSpec{ WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, - WorkflowOwner: wfOwnerHex, + WorkflowOwner: "aabbccdd", }, }, newMeteringResourceManager(t, true, emitter)) @@ -392,7 +370,7 @@ func Test_meterRecords(t *testing.T) { assert.Empty(t, emitter.Records()) }) - t.Run("snapshot emits one MeterSnapshot per running engine", func(t *testing.T) { + t.Run("snapshot emits one MeterSnapshot per persisted spec", func(t *testing.T) { t.Parallel() emitter := &recordingSnapshotEmitter{} clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) @@ -403,16 +381,20 @@ func Test_meterRecords(t *testing.T) { SnapshotInterval: time.Minute, Clock: clock, }) - h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, rm) - unregister := rm.Register(h) - t.Cleanup(unregister) - // Register two engines; the running engine registry is the in-memory source - // the snapshot reads (no per-tick GetWorkflowSpec). + // The snapshot enumerates persisted specs from the store, not running + // engines, so paused-but-stored specs are still accounted for. wfID1 := types.WorkflowID{20} wfID2 := types.WorkflowID{21} - require.NoError(t, h.engineRegistry.Add(wfID1, "test-source", &fakeService{})) - require.NoError(t, h.engineRegistry.Add(wfID2, "test-source", &fakeService{})) + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{ + {WorkflowID: wfID1.Hex(), WorkflowOwner: "aabbccdd"}, + {WorkflowID: wfID2.Hex(), WorkflowOwner: "aabbccdd"}, + }, + } + h := newMeteringTestHandler(t, store, rm) + unregister := rm.Register(h) + t.Cleanup(unregister) servicetest.Run(t, rm) require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) @@ -434,23 +416,7 @@ func Test_meterRecords(t *testing.T) { // resource_id = workflow_id fully identifies the resource; no labels. byWorkflowID[snap.Utilization[0].ResourceId] = snap } - require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first running engine") - require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second running engine") - }) - - t.Run("graceful close emits a RELEASE per running engine", func(t *testing.T) { - t.Parallel() - emitter := &recordingEmitter{} - h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) - - wfID := types.WorkflowID{30} - require.NoError(t, h.engineRegistry.Add(wfID, "test-source", &fakeService{})) - - es := h.engineRegistry.GetAll() - h.emitGracefulCloseReleases(t.Context(), es) - - records := emitter.Records() - require.Len(t, records, 1) - requireMeterRecord(t, records[0], meteringpb.MeterAction_METER_ACTION_RELEASE, WorkflowDeleted, wfID.Hex()) + require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first persisted spec") + require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second persisted spec") }) } diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 8dc521088b7..dca270b6114 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -1025,6 +1025,10 @@ func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflo return m.artifactStore.DeleteWorkflowArtifacts(ctx, workflowID) } +func (m *mockArtifactStore) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return m.artifactStore.GetWorkflowSpecList(ctx) +} + func (m *mockArtifactStore) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { return m.artifactStore.DeleteWorkflowArtifactsBatch(ctx, workflowIDs) } @@ -1298,8 +1302,10 @@ func Test_workflowDeletedHandler(t *testing.T) { type stubWorkflowArtifactsStore struct { spec *job.WorkflowSpec + specs []*job.WorkflowSpec upsertErr error deleteErr error + listErr error deleteCalls atomic.Int32 } @@ -1321,6 +1327,13 @@ func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(context.Context, *job.Wo return 1, nil } +func (s *stubWorkflowArtifactsStore) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + if s.listErr != nil { + return nil, s.listErr + } + return s.specs, nil +} + func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) error { s.deleteCalls.Add(1) return s.deleteErr diff --git a/core/services/workflows/syncer/v2/mocks/orm.go b/core/services/workflows/syncer/v2/mocks/orm.go index a4d02addf29..67c36249cd5 100644 --- a/core/services/workflows/syncer/v2/mocks/orm.go +++ b/core/services/workflows/syncer/v2/mocks/orm.go @@ -175,6 +175,64 @@ func (_c *ORM_GetWorkflowSpec_Call) RunAndReturn(run func(context.Context, strin return _c } +// GetWorkflowSpecList provides a mock function with given fields: ctx +func (_m *ORM) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetWorkflowSpecList") + } + + var r0 []*job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*job.WorkflowSpec, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*job.WorkflowSpec); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*job.WorkflowSpec) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ORM_GetWorkflowSpecList_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetWorkflowSpecList' +type ORM_GetWorkflowSpecList_Call struct { + *mock.Call +} + +// GetWorkflowSpecList is a helper method to define mock.On call +// - ctx context.Context +func (_e *ORM_Expecter) GetWorkflowSpecList(ctx interface{}) *ORM_GetWorkflowSpecList_Call { + return &ORM_GetWorkflowSpecList_Call{Call: _e.mock.On("GetWorkflowSpecList", ctx)} +} + +func (_c *ORM_GetWorkflowSpecList_Call) Run(run func(ctx context.Context)) *ORM_GetWorkflowSpecList_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ORM_GetWorkflowSpecList_Call) Return(_a0 []*job.WorkflowSpec, _a1 error) *ORM_GetWorkflowSpecList_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ORM_GetWorkflowSpecList_Call) RunAndReturn(run func(context.Context) ([]*job.WorkflowSpec, error)) *ORM_GetWorkflowSpecList_Call { + _c.Call.Return(run) + return _c +} + // UpsertWorkflowSpec provides a mock function with given fields: ctx, spec func (_m *ORM) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { ret := _m.Called(ctx, spec) diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index cb9ed7589f7..31673d9262d 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -367,7 +367,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 66a6e5a19fd..833777f323c 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -390,7 +390,7 @@ Tenant = 'mainline' NumericTenantID = '42' Environment = 'production' Zone = 'wf-zone-a' -NodeID = 'csa-pubkey-1' +NodeID = 'clp-cre-wf-zone-a-1' [Workflows] [Workflows.Limits] diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index c7a8e8dc3eb..e665f1c7452 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -367,7 +367,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/deployment/go.mod b/deployment/go.mod index d380046bfbd..d29fef9949c 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -5,6 +5,13 @@ go 1.26.4 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ +// Local development replaces for the delta-based metering redesign, kept +// consistent with the root module. The maintainer drops these and bumps the +// module pins at merge time. +replace github.com/smartcontractkit/chainlink-common => ../../chainlink-common + +replace github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go + // chainlink-evm's generated codecgen is incompatible with ugorji/go/codec v1.3.1 // (pulled in transitively by mcms v0.47.x). Pin to the version the rest of the repo uses. replace github.com/ugorji/go/codec => github.com/ugorji/go/codec v1.2.12 @@ -441,7 +448,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 3824cdbb1a7..265c715e54b 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1435,6 +1435,8 @@ github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-8 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index b5a8cb50c0d..c122d0e6c20 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2571,7 +2571,7 @@ By default, we only forward the go runtime metrics. Empty means forward everythi [Metering] MeterRecordsEnabled = false # Default MeterSnapshotsEnabled = false # Default -Product = '' # Default +Product = 'cre' # Default Tenant = '' # Default NumericTenantID = '' # Default Environment = '' # Default @@ -2580,7 +2580,13 @@ NodeID = '' # Default ``` Metering configures durable resource metering emission and the coarse deployment/node identity dimensions stamped on emitted MeterRecords and -MeterSnapshots. These are plumbed to LOOP plugins via env. +MeterSnapshots. This TOML section is the single authority for metering on the +core node; there is no environment-variable gate. For capability LOOP plugins +the values are forwarded unchanged over the plugin environment +(loop.EnvConfig), which is only a child-process transport produced from this +config, not a separate gate. Snapshots are emitted on a fixed internal +interval and are bucket-aligned (each snapshot timestamp is truncated to the +interval) so cross-node snapshot buckets agree. ### MeterRecordsEnabled ```toml @@ -2597,7 +2603,7 @@ Requires MeterRecordsEnabled = true. ### Product ```toml -Product = '' # Default +Product = 'cre' # Default ``` Product is the deployment product identity dimension, e.g. 'cre'. diff --git a/go.mod b/go.mod index 78218df496e..66f56ba9a54 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 @@ -436,3 +436,12 @@ tool github.com/smartcontractkit/chainlink-common/pkg/loop/cmd/loopinstall tool github.com/smartcontractkit/chainlink-common/script/cmd/dependabot replace github.com/smartcontractkit/chainlink-sui => github.com/smartcontractkit/chainlink-sui v0.0.0-20260707125635-abec997b6eae + +// Local development replaces for the delta-based metering redesign. The +// chainlink-common (feat/metering-v2-cll-meter) and chainlink-protos +// (feat/metering-v2-cll-meter) siblings carry the new resourcemanager API and +// metering protos this branch builds against. The maintainer drops these and +// bumps the module pins at merge time. +replace github.com/smartcontractkit/chainlink-common => ../chainlink-common + +replace github.com/smartcontractkit/chainlink-protos/metering/go => ../chainlink-protos/metering/go diff --git a/go.sum b/go.sum index 1d03191a326..cedc7d236c2 100644 --- a/go.sum +++ b/go.sum @@ -1208,6 +1208,8 @@ github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-8 github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0/go.mod h1:m/A3lqD7ms/RsQ9BT5P2uceYY0QX5mIt4KQxT2G6qEo= github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd h1:7DURXB3+Qf9REr3XA+q0FNyZO3CSAeSgJvNaek/GiZI= diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index ede01b4ed87..2f4d523dc9a 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -101,7 +101,7 @@ func (m mockCfgMetering) Tenant() string { return "mainline" } func (m mockCfgMetering) NumericTenantID() string { return "42" } func (m mockCfgMetering) Environment() string { return "production" } func (m mockCfgMetering) Zone() string { return "wf-zone-a" } -func (m mockCfgMetering) NodeID() string { return "csa-pubkey-1" } +func (m mockCfgMetering) NodeID() string { return "clp-cre-wf-zone-a-1" } type mockPrometheusBridge struct{} @@ -270,7 +270,7 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, "42", envCfg.MeterNumericTenantID) require.Equal(t, "production", envCfg.MeterEnvironment) require.Equal(t, "wf-zone-a", envCfg.MeterZone) - require.Equal(t, "csa-pubkey-1", envCfg.MeterNodeID) + require.Equal(t, "clp-cre-wf-zone-a-1", envCfg.MeterNodeID) require.Equal(t, "example.com/chip-ingress", envCfg.ChipIngressEndpoint) require.False(t, envCfg.ChipIngressBatchEmitterEnabled) diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 3b74face983..bb429faa2da 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -514,7 +514,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index 9e0f05c374c..90a5d296ee2 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -379,7 +379,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 5e7958e30a7..538ee62957a 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -440,7 +440,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index a6aec07ebde..d6a1dd3bc3f 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -423,7 +423,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 47e3d90fc7a..11b0a412f91 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -423,7 +423,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 37ed971264b..bb1e0a47757 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -423,7 +423,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 629b75c1c10..5b1c20bbcbc 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -525,7 +525,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index 488f4199388..9f002014e57 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -408,7 +408,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index cec424c990c..16bf408adab 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -419,7 +419,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index c8b9da305ce..ee59bf876df 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -420,7 +420,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 32598b0d36e..1da01906508 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -402,7 +402,9 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = '' +Product = 'cre' +Tenant = '' +NumericTenantID = '' Environment = '' Zone = '' NodeID = '' From 1898360ef163d256101c38bf645c0484896582c9 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 10 Jul 2026 14:10:28 -0400 Subject: [PATCH 06/14] feat(metering): supply deterministic cross-node event_id from workflow syncer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The common EmitDelta now requires a producer-supplied event_id. The syncer is a workflow-DON service driven by reconciliation against the shared on-chain WorkflowRegistry, so every workflow-DON node sees the same event and derives the identical id via resourcemanager.EventID: - create (+1): EventID("workflow-spec-register", workflowID, CreatedAt) — the on-chain CreatedAt is DON-consistent and distinguishes a re-registration from the original (re-activate-after-delete does not collide). - delete (-1): EventID("workflow-spec-delete", workflowID). WorkflowDeletedEvent carries only the workflow ID (no on-chain CreatedAt/block), so repeated delete cycles of the same workflowID share an event_id; this residual drift is bounded by snapshot reconciliation. See the returned blocker note. Scrubs the "UUIDv4" event_id wording and the UUID-format validation from the metering test (event_id is an opaque string; the only check is empty-vs-nonempty). Aligns the metering pin to the merged chainlink-common pin. --- core/services/workflows/syncer/v2/handler.go | 27 ++++++++---- .../syncer/v2/handler_metering_test.go | 41 +++++++++++-------- go.mod | 2 +- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 5f47c07fb95..36caf269fb1 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -683,7 +683,8 @@ func (h *eventHandler) workflowRegisteredEvent( } // A new spec's artifacts were persisted for the first time: the durable // workflow_specs_v2 resource gained one unit, so emit a +1 delta. - h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner)) + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) spec = newSpec case spec.WorkflowID != payload.WorkflowID.Hex(): @@ -693,7 +694,8 @@ func (h *eventHandler) workflowRegisteredEvent( } // A different spec's artifacts were persisted under this key: the newly // stored spec is a fresh durable resource, so emit a +1 delta. - h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner)) + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) spec = newSpec case spec.Status != status: @@ -1017,7 +1019,12 @@ func (h *eventHandler) workflowDeletedEvent( // realized by the spec's absence from subsequent snapshots, so pause emits // no delta. if originatingEvent == WorkflowDeleted { - h.emitSpecDelta(ctx, -1, workflowID, deleteOwner) + // WorkflowDeletedEvent carries only the workflow ID (no on-chain + // CreatedAt/block), so the delete event_id is derived from the + // DON-consistent workflowID alone. See the emitSpecDelta call for the + // re-delete-cycle limitation noted in the metering redesign. + h.emitSpecDelta(ctx, -1, workflowID, deleteOwner, + resourcemanager.EventID("workflow-spec-delete", workflowID)) } h.cleanupModuleCache(payload.WorkflowID.Hex()) @@ -1049,16 +1056,20 @@ func (h *eventHandler) workflowDeletedEvent( // owner is the workflow owner as stored in durable state; the organization is // resolved fail-open at emit time via ResolveOrEmpty and is never persisted. // Emission is fail-open and returns no error, so metering can never gate, -// delay, or fail the storage operation it records. event_id is generated per -// emission by the ResourceManager (a fresh UUIDv4) and is the consumer's dedup -// key. -func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowID, owner string) { +// delay, or fail the storage operation it records. +// +// eventID is the caller-supplied, deterministic cross-node dedup key. The syncer +// is a workflow-DON service driven by reconciliation against the shared on-chain +// WorkflowRegistry, so every workflow-DON node sees the same reconciliation +// event and derives the identical id from it (see the call sites). It is never +// generated here; an empty eventID is rejected by the ResourceManager. +func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowID, owner, eventID string) { if h.resourceManager == nil { return } orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) // resource_id = workflow_id (the syncer meters one durable spec per workflow). - h.resourceManager.EmitDelta(ctx, h.baseIdentity(), delta, resourcemanager.UtilizationFields{ + h.resourceManager.EmitDelta(ctx, h.baseIdentity(), eventID, delta, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, ResourceID: workflowID, OrgID: orgID, diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index 90d6789e798..6c4ea25f067 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -136,9 +135,10 @@ func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, // requireSpecDelta asserts that record is a workflow-syncer-v2 spec delta: a // METER_ACTION_UPDATE carrying a single utilization with the given signed value -// and resource_id (= workflow_id). event_id must be a manager-generated UUID, -// not the originating event name. -func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workflowID string) { +// and resource_id (= workflow_id). event_id must be the deterministic, +// cross-node-identical id wantEventID (an opaque string that is never +// format-validated). +func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workflowID, wantEventID string) { t.Helper() require.NotNil(t, record.Identity) assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) @@ -153,9 +153,9 @@ func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workf assert.Equal(t, "operations", util.ResourceType) // resource_id = workflow_id for the syncer (no shared physical resource). assert.Equal(t, workflowID, util.ResourceId) - // event_id is generated per emission by the ResourceManager (a UUIDv4). - _, parseErr := uuid.Parse(util.EventId) - assert.NoError(t, parseErr, "event_id must be a manager-generated UUID, got %q", util.EventId) + // event_id is the deterministic reconciliation-derived id, identical on every + // workflow-DON node; it is an opaque string with no validated format. + assert.Equal(t, wantEventID, util.EventId) } func Test_meterRecords(t *testing.T) { @@ -179,14 +179,16 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireSpecDelta(t, records[0], "1", wfID.Hex()) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), "0")) }) - t.Run("retried registered event emits a fresh event_id per emission", func(t *testing.T) { + t.Run("reprocessed registered event emits the IDENTICAL event_id (cross-node dedup)", func(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} // The stub never returns a stored spec, so each call replays the - // new-spec path exactly as a reprocessed event would. + // new-spec path exactly as a reprocessed event (or a second node + // reconciling the same on-chain event) would. h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) event := WorkflowRegisteredEvent{ @@ -194,6 +196,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: types.WorkflowID{2}, WorkflowOwner: wfOwner, WorkflowName: "wf-name", + CreatedAt: 123, } require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) @@ -202,12 +205,12 @@ func Test_meterRecords(t *testing.T) { require.Len(t, records, 2) require.Len(t, records[0].Utilizations, 1) require.Len(t, records[1].Utilizations, 1) - assert.Equal(t, records[0].Action, records[1].Action) - assert.Equal(t, records[0].Identity.GetService(), records[1].Identity.GetService()) - assert.Equal(t, records[0].Identity.GetResourcePool(), records[1].Identity.GetResourcePool()) - assert.Equal(t, records[0].Utilizations[0].GetResourceId(), records[1].Utilizations[0].GetResourceId()) - // event_id is generated fresh per emission; retries do not reuse it. - assert.NotEqual(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) + // The same reconciliation event MUST yield the identical event_id, so the + // billing consumer dedups reprocessing and cross-node duplicates. It is + // derived deterministically from the on-chain workflowID + CreatedAt. + want := resourcemanager.EventID("workflow-spec-register", types.WorkflowID{2}.Hex(), "123") + assert.Equal(t, want, records[0].Utilizations[0].GetEventId()) + assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) }) t.Run("activating an already-stored spec emits nothing (status-only update)", func(t *testing.T) { @@ -270,7 +273,8 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireSpecDelta(t, records[0], "-1", wfID.Hex()) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) }) t.Run("no record when persisting a new spec fails", func(t *testing.T) { @@ -331,7 +335,8 @@ func Test_meterRecords(t *testing.T) { records := emitter.Records() require.Len(t, records, 1) - requireSpecDelta(t, records[0], "-1", wfID.Hex()) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) }) t.Run("emit failure never fails event handling", func(t *testing.T) { diff --git a/go.mod b/go.mod index 66f56ba9a54..59ee03473aa 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260623200841-e0322b819f62 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd From 58607d9aa8f5017c510fb0a3b78a2bfb5ce4839a Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 15 Jul 2026 14:05:45 -0400 Subject: [PATCH 07/14] updates from chainlink-common changes --- core/scripts/go.mod | 4 ++-- core/scripts/go.sum | 6 ------ core/services/cre/cre.go | 12 ++---------- core/services/workflows/syncer/v2/handler.go | 18 +++++++++++++++--- deployment/go.mod | 4 ++-- deployment/go.sum | 6 ------ go.mod | 2 +- go.sum | 6 ------ integration-tests/go.mod | 6 +++--- integration-tests/go.sum | 12 ++++++------ integration-tests/load/go.mod | 6 +++--- integration-tests/load/go.sum | 12 ++++++------ system-tests/lib/go.mod | 6 +++--- system-tests/lib/go.sum | 12 ++++++------ system-tests/tests/go.mod | 6 +++--- system-tests/tests/go.sum | 12 ++++++------ 16 files changed, 58 insertions(+), 72 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 493f97c777d..73a886bf6c4 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -54,7 +54,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -514,7 +514,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 1167d62f457..87f0627ec70 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1580,8 +1580,6 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1626,10 +1624,6 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index df8c192180c..f5ef7a37b73 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -244,16 +244,8 @@ func (s *Services) newSubservices( return nil, fmt.Errorf("could not create org resolver: %w", ierr) } fallbackResolver := orgresolver.NewOrgResolverWithFallback(inner, lggr) - // Wrap in a caching resolver so the metering snapshot path - // (Meterable.GetUtilization, which is contractually no-network) can - // resolve org attribution from memory. This single instance is shared: - // the syncer's record path resolves through it too, warming the cache - // for snapshots. The caching resolver owns the inner resolver's - // lifecycle (its Start/Close drive the inner), so only the caching - // resolver is added to the service list. - cachingResolver := orgresolver.NewCaching(fallbackResolver, resourcemanager.DefaultSnapshotInterval) - s.OrgResolver = cachingResolver - srvs = append(srvs, cachingResolver) + s.OrgResolver = fallbackResolver + srvs = append(srvs, fallbackResolver) } else { lggr.Warn("Skipping orgResolver, no linking service configured") } diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 36caf269fb1..f5bc29c9916 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -1054,7 +1054,7 @@ func (h *eventHandler) workflowDeletedEvent( // the workflow-spec storage mutation has committed. // // owner is the workflow owner as stored in durable state; the organization is -// resolved fail-open at emit time via ResolveOrEmpty and is never persisted. +// resolved fail-open at emit time and is never persisted. // Emission is fail-open and returns no error, so metering can never gate, // delay, or fail the storage operation it records. // @@ -1067,7 +1067,14 @@ func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowI if h.resourceManager == nil { return } - orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, owner, h.lggr) + var orgID string + if h.orgResolver != nil && owner != "" { + if resolved, err := h.orgResolver.Get(ctx, owner); err != nil { + h.lggr.Warnw("failed to resolve org ID for metering", "owner", owner, "err", err) + } else { + orgID = resolved + } + } // resource_id = workflow_id (the syncer meters one durable spec per workflow). h.resourceManager.EmitDelta(ctx, h.baseIdentity(), eventID, delta, resourcemanager.UtilizationFields{ ResourceType: meterResourceType, @@ -1128,7 +1135,12 @@ func (h *eventHandler) GetUtilization(ctx context.Context) []resourcemanager.Sna if spec == nil { continue } - orgID := orgresolver.ResolveOrEmpty(ctx, h.orgResolver, spec.WorkflowOwner, h.lggr) + var orgID string + if h.orgResolver != nil && spec.WorkflowOwner != "" { + if resolved, err := h.orgResolver.Get(ctx, spec.WorkflowOwner); err == nil { + orgID = resolved + } + } entries = append(entries, resourcemanager.SnapshotEntry{ Identity: base, Utilizations: []*meteringpb.Utilization{ diff --git a/deployment/go.mod b/deployment/go.mod index d29fef9949c..ef3b54828aa 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -54,7 +54,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -447,7 +447,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 265c715e54b..d5b3ad98383 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1385,8 +1385,6 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1431,10 +1429,6 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/go.mod b/go.mod index 59ee03473aa..ad4195ee72f 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260522094612-5f9f748bd87a diff --git a/go.sum b/go.sum index cedc7d236c2..34fde95f8b8 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,6 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1204,10 +1202,6 @@ github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546- github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index efe2be9ea9a..9d61ad6e6b8 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -33,7 +33,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae @@ -61,7 +61,7 @@ require ( ) require ( - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260706170510-94f3a0409ea5 // indirect ) @@ -425,7 +425,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260623200841-e0322b819f62 // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 09f39fdb0c3..a0d99fd9f72 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1372,8 +1372,8 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1418,10 +1418,10 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 12f549a887e..8610243cffc 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.6-0.20260630120514-36abe27604df @@ -502,8 +502,8 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index ea97729af9b..dcf4ca9d6fa 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1634,8 +1634,8 @@ github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1680,10 +1680,10 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index c6a8366bfd4..d2398b2b1c3 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260706100550-d43558069754 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260623170329-4577ef4ba0ae @@ -473,8 +473,8 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 0829f3bbc41..4f6c1ea3b3e 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1545,8 +1545,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1591,10 +1591,10 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index d1b522c2d62..82909ae894c 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -62,7 +62,7 @@ require ( github.com/rs/zerolog v1.35.1 github.com/smartcontractkit/chain-selectors v1.0.104 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba github.com/smartcontractkit/chainlink-common/keystore v1.2.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -165,8 +165,8 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 // indirect - github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260706170510-94f3a0409ea5 // indirect github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.9 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 7895b293a88..9b0e5dc81b3 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1559,8 +1559,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094 h1:uMn1w/n95p+pSxK6hYNqji/sDaab8D0Cxuph9qUkM2g= github.com/smartcontractkit/chainlink-ccv v0.0.2-0.20260706093831-dad26b360094/go.mod h1:3XQmy2zQHrmufebP7dM+x595+gghxzyIKxK9wv91Rh8= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6 h1:uvUnQlWgJGJBgtXpxpgGTipUZPxp/JugxBK7KUe41DQ= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260706203129-712bad3efbc6/go.mod h1:2hczeHjBapPah8d3EOoRAfzI4gH3caZToMhAUG8WS3I= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.0 h1:1BH/b14CkGjArfzznlioQpIJiynECWVT48JUP9E277U= github.com/smartcontractkit/chainlink-common/keystore v1.2.0/go.mod h1:9R/74vN+bJ5PbkOyM/pUy/AeAZaRwYb/k4XPeXcbDio= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1605,10 +1605,10 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0 h1:FC+WdJ8Y github.com/smartcontractkit/chainlink-protos/job-distributor v0.19.0/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305 h1:bnSl5p3mFekSJ6QcbZ1TmHn2ffYiX8xk6hNzVmyhstQ= -github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260512230622-65f10f4cd305/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= +github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0/go.mod h1:PjZD54vr6rIKEKQj6HNA4hllvYI/QpT+Zefj3tqkFAs= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= From a500659da1a776c67c51a4922bb294b9971e1b21 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 15 Jul 2026 17:56:35 -0400 Subject: [PATCH 08/14] lint --- .../chainlink/config_metering_test.go | 19 +++++++++++-------- core/services/chainlink/config_test.go | 16 ++++++++-------- .../standard_capabilities_test.go | 4 ++++ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go index 093cb806c66..de330c012cf 100644 --- a/core/services/chainlink/config_metering_test.go +++ b/core/services/chainlink/config_metering_test.go @@ -9,7 +9,9 @@ import ( ) func TestMeteringConfig(t *testing.T) { + t.Parallel() t.Run("defaults", func(t *testing.T) { + t.Parallel() mc := meteringConfig{s: toml.Metering{}} assert.False(t, mc.MeterRecordsEnabled()) assert.False(t, mc.MeterSnapshotsEnabled()) @@ -25,15 +27,16 @@ func TestMeteringConfig(t *testing.T) { }) t.Run("explicit values", func(t *testing.T) { + t.Parallel() mc := meteringConfig{s: toml.Metering{ - MeterRecordsEnabled: ptr(true), - MeterSnapshotsEnabled: ptr(true), - Product: ptr("cre"), - Tenant: ptr("mainline"), - NumericTenantID: ptr("42"), - Environment: ptr("production"), - Zone: ptr("wf-zone-a"), - NodeID: ptr("clp-cre-wf-zone-a-1"), + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), }} assert.True(t, mc.MeterRecordsEnabled()) assert.True(t, mc.MeterSnapshotsEnabled()) diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index da17465bf39..8e8657281be 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -582,14 +582,14 @@ func TestConfig_Marshal(t *testing.T) { }, } full.Metering = toml.Metering{ - MeterRecordsEnabled: ptr(true), - MeterSnapshotsEnabled: ptr(true), - Product: ptr("cre"), - Tenant: ptr("mainline"), - NumericTenantID: ptr("42"), - Environment: ptr("production"), - Zone: ptr("wf-zone-a"), - NodeID: ptr("clp-cre-wf-zone-a-1"), + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), } full.CRE = toml.CreConfig{ UseLocalTimeProvider: ptr(true), diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index d24096b6fac..ee375120bce 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -108,6 +108,7 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { } func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { + t.Parallel() want := core.StandardCapabilitiesDependencies{ Config: "test-config", } @@ -131,7 +132,9 @@ func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { // falls back to the consumer workflow's DON for metering identity and event // labels even when the node knows which DON it is serving. func TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP(t *testing.T) { + t.Parallel() t.Run("nonzero DON ID round-trips when the DON is known", func(t *testing.T) { + t.Parallel() const knownDonID = uint32(42) std := NewStandardCapabilities( logger.TestLogger(t), @@ -148,6 +151,7 @@ func TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP(t *testing.T) { }) t.Run("zero DON ID is preserved when the host could not resolve one", func(t *testing.T) { + t.Parallel() std := NewStandardCapabilities( logger.TestLogger(t), "not/found/path/to/binary", From cc1ab09835aa4559525f7a8f3f16dd003fcc8904 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 15 Jul 2026 18:10:36 -0400 Subject: [PATCH 09/14] removing local replaces --- core/scripts/go.mod | 7 ------- core/scripts/go.sum | 4 ++++ deployment/go.mod | 7 ------- deployment/go.sum | 4 ++++ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index b2fc1001684..7ded20dcc73 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -5,13 +5,6 @@ go 1.26.4 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ -// Local development replaces for the delta-based metering redesign, kept -// consistent with the root module. The maintainer drops these and bumps the -// module pins at merge time. -replace github.com/smartcontractkit/chainlink-common => ../../../chainlink-common - -replace github.com/smartcontractkit/chainlink-protos/metering/go => ../../../chainlink-protos/metering/go - replace github.com/smartcontractkit/chainlink/deployment => ../../deployment replace github.com/smartcontractkit/chainlink/system-tests/lib => ../../system-tests/lib diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 9629695f6e5..70c38662439 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1580,6 +1580,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h1:sH546tdm3VaYGaRLnZ6ZYBZiE25vEkMap76Xq/vQWx0= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1624,6 +1626,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/deployment/go.mod b/deployment/go.mod index af438176b76..b39e7d8e17d 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -5,13 +5,6 @@ go 1.26.4 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ -// Local development replaces for the delta-based metering redesign, kept -// consistent with the root module. The maintainer drops these and bumps the -// module pins at merge time. -replace github.com/smartcontractkit/chainlink-common => ../../chainlink-common - -replace github.com/smartcontractkit/chainlink-protos/metering/go => ../../chainlink-protos/metering/go - // chainlink-evm's generated codecgen is incompatible with ugorji/go/codec v1.3.1 // (pulled in transitively by mcms v0.47.x). Pin to the version the rest of the repo uses. replace github.com/ugorji/go/codec => github.com/ugorji/go/codec v1.2.12 diff --git a/deployment/go.sum b/deployment/go.sum index 32da3d2a4d5..f3620bf625d 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1390,6 +1390,8 @@ github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= @@ -1434,6 +1436,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= From 1b333bec1a981935b86d20c89f3836a0f530598a Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 16 Jul 2026 18:32:32 -0400 Subject: [PATCH 10/14] cleanup --- core/config/docs/core.toml | 8 +------- core/config/toml/types.go | 4 +--- core/services/chainlink/config_metering.go | 7 +++---- core/services/chainlink/config_metering_test.go | 2 +- .../chainlink/testdata/config-empty-effective.toml | 2 +- .../testdata/config-multi-chain-effective.toml | 2 +- core/services/cre/cre.go | 7 +------ .../standardcapabilities/standard_capabilities.go | 4 ---- .../standard_capabilities_test.go | 3 +-- core/services/workflows/syncer/v2/handler.go | 11 ++--------- plugins/plugins.private.yaml | 6 +++--- 11 files changed, 15 insertions(+), 41 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 8a0d840ab7f..a24424881c9 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -931,13 +931,7 @@ Prefixes = ["go_"] # Default # Metering configures durable resource metering emission and the coarse # deployment/node identity dimensions stamped on emitted MeterRecords and -# MeterSnapshots. This TOML section is the single authority for metering on the -# core node; there is no environment-variable gate. For capability LOOP plugins -# the values are forwarded unchanged over the plugin environment -# (loop.EnvConfig), which is only a child-process transport produced from this -# config, not a separate gate. Snapshots are emitted on a fixed internal -# interval and are bucket-aligned (each snapshot timestamp is truncated to the -# interval) so cross-node snapshot buckets agree. +# MeterSnapshots. [Metering] # MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. MeterRecordsEnabled = false # Default diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 33b8dc54150..14b4be5cc48 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -3118,9 +3118,7 @@ type Metering struct { // Zone is the deployment zone dimension, e.g. "wf-zone-a". Zone *string // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1" (NOT the CSA - // public key). The billing service uses it to look up the node's CSA key in - // the workflow registry; the CSA key itself is attached to events as the - // node_csa_key attribute. + // public key) NodeID *string } diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go index ac8bd147887..632b2b1fa3b 100644 --- a/core/services/chainlink/config_metering.go +++ b/core/services/chainlink/config_metering.go @@ -22,12 +22,11 @@ func (b *meteringConfig) MeterSnapshotsEnabled() bool { return *b.s.MeterSnapshotsEnabled } -// Product defaults to "cre" (resourcemanager.DefaultMeteringProduct) when unset, -// so the syncer's metering identity matches the capability plugins' fallback and -// metering can never be enabled with an empty product dimension. +// Product defaults to "unset" (resourcemanager.DefaultMeteringProduct) when unset, +// to catch config errors early. func (b *meteringConfig) Product() string { if b.s.Product == nil { - return "cre" + return "unset" } return *b.s.Product } diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go index de330c012cf..42f64260f8c 100644 --- a/core/services/chainlink/config_metering_test.go +++ b/core/services/chainlink/config_metering_test.go @@ -18,7 +18,7 @@ func TestMeteringConfig(t *testing.T) { // Product defaults to "cre" so metering can never be enabled with an // empty product dimension and the syncer identity matches the plugins' // fallback. - assert.Equal(t, "cre", mc.Product()) + assert.Equal(t, "unset", mc.Product()) assert.Empty(t, mc.Tenant()) assert.Empty(t, mc.NumericTenantID()) assert.Empty(t, mc.Environment()) diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 0b5b7d4539d..7d607891451 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -370,7 +370,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = 'cre' +Product = 'unset' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index e2879ba1cba..55f9cbe7314 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -370,7 +370,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = 'cre' +Product = 'unset' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index f5ef7a37b73..10095699655 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -1042,8 +1042,7 @@ func newWorkflowRegistrySyncerV2( } shardRoutingSteady = shardownership.NewSteadySignal(shardownership.WithSteadySignalMetrics(steadyMetrics)) } - // TOML [Metering] is the single authority for the emission gate; the - // deleted CL_METER_RECORDS_ENABLED node env var no longer participates. + meteringCfg := cfg.Metering() meterRecordsEnabled := meteringCfg != nil && meteringCfg.MeterRecordsEnabled() meterSnapshotsEnabled := meteringCfg != nil && meteringCfg.MeterSnapshotsEnabled() @@ -1056,10 +1055,6 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithLocalSecretOverrides(lggr, cfg.CRE().LocalSecretOverrides()), syncerV2.WithShardExecutionGuard(shardOrchestratorClient, shardingEnabled, shardIndex), syncerV2.WithShardRoutingSteady(shardRoutingSteady), - // The handler owns this ResourceManager's lifecycle (starts it, registers - // itself as the snapshotted Meterable, closes it). A positive - // SnapshotInterval enables the periodic absolute-state snapshot loop; the - // RM is otherwise a no-op when metering is disabled. syncerV2.WithResourceManager(resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ MeterRecordsEnabled: meterRecordsEnabled, MeterSnapshotsEnabled: meterSnapshotsEnabled, diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index af549670559..c9ec5d99a63 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -106,10 +106,6 @@ func (s *StandardCapabilities) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to parse capabilities env file: %w", err) } - // Metering identity and the emission gate reach capability LOOPs through - // loop.EnvConfig (produced from the node's TOML [Metering] via - // EnvConfig.AsCmdEnv), so there is no meter-record env var to pass - // through here. cmdFn, opts, err := s.pluginRegistrar.RegisterLOOP(plugins.CmdConfig{ ID: s.log.Name(), Cmd: s.command, diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index ee375120bce..72094b5b172 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -80,8 +80,7 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { require.Contains(t, err.Error(), capturingRegistrarErr) require.Empty(t, cfg.Env, - "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset; "+ - "metering config reaches LOOPs via loop.EnvConfig, not this pass-through") + "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset; ") }) t.Run("missing env file fails Start before RegisterLOOP", func(t *testing.T) { diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index cab51375a8e..85c160cbca3 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -105,14 +105,7 @@ type eventHandler struct { workflowDonSubscriber capabilities.DonSubscriber billingClient metering.BillingClient resourceManager *resourcemanager.ResourceManager - // meterIdentity is the base metering identity for this node's syncer: the - // six coarse dimensions (product, tenant, environment, zone, don_id, - // node_id) plus service/resource_pool. Per-resource billing fields - // (resource_type/resource_id/event_id/org_id/value) are attached at emit/ - // snapshot time via UtilizationFields. It is populated by WithIdentity in - // cre.go; Service/ResourcePool are forced to syncer constants regardless of - // the option. - meterIdentity resourcemanager.ResourceIdentity + meterIdentity resourcemanager.ResourceIdentity // resolvedDonID holds the workflow DON id once resolved from the don notifier // at start (the engine runs on the workflow DON). It is resolved // asynchronously so node boot is not blocked while waiting for the DON to be @@ -374,7 +367,6 @@ func NewEventHandler( Service: meterService, ResourcePool: meterResourcePool, }, - tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode } metricsInst, metricsErr := newMetrics() if metricsErr != nil { @@ -407,6 +399,7 @@ func (h *eventHandler) start(ctx context.Context) error { if err := h.resourceManager.Start(ctx); err != nil { return fmt.Errorf("failed to start resource manager: %w", err) } + // Register returns a func that unregisters. h.rmUnregister = h.resourceManager.Register(h) h.resolveWorkflowDonID() } diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 0f4683230b0..24d0bfa8eff 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -9,7 +9,7 @@ defaults: plugins: cron: - moduleURI: "github.com/smartcontractkit/capabilities/cron" - gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + gitRef: "792fd475a9c294b61fa90b9dd23388da2827f093" installPath: "." flags: "-tags timetzdata" readcontract: @@ -31,11 +31,11 @@ plugins: installPath: "." httptrigger: - moduleURI: "github.com/smartcontractkit/capabilities/http_trigger" - gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + gitRef: "2cf965790a71171e1336f6d1d4f5162f2bd39bd1" installPath: "." evm: - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/evm" - gitRef: "38038ef2e97d15e74491df228460f182c8cb81a5" + gitRef: "d7162a2b15aea2ce861052ebaa2f5ca716a9b60a" installPath: "." solana: - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/solana" From 3d48b1cba7a337400acad27e8f3d220959b72805 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 30 Jul 2026 15:32:45 -0400 Subject: [PATCH 11/14] dependency bumps, refactoring, some correctness changes --- GNUmakefile | 2 +- core/config/toml/types.go | 7 + core/config/toml/types_test.go | 62 +++++ .../workflow-gateway-capabilities-don.toml | 189 ++++++++++++++- .../environment/chip_ingress_stack.go | 18 +- core/scripts/go.mod | 10 +- core/scripts/go.sum | 20 +- core/services/chainlink/config_metering.go | 6 +- .../chainlink/config_metering_test.go | 8 +- .../chainlink/mocks/general_config.go | 94 ++++---- .../testdata/config-empty-effective.toml | 2 +- .../config-multi-chain-effective.toml | 2 +- core/services/workflows/artifacts/v2/orm.go | 1 + core/services/workflows/syncer/v2/handler.go | 119 +++++---- .../syncer/v2/handler_metering_test.go | 227 +++++++++++++++++- .../workflows/syncer/v2/handler_test.go | 49 ++-- core/services/workflows/syncer/v2/helpers.go | 5 + .../workflows/syncer/v2/workflow_registry.go | 70 ++++++ .../syncer/v2/workflow_registry_test.go | 61 +++++ deployment/go.mod | 10 +- deployment/go.sum | 20 +- docs/CONFIG.md | 1 + go.mod | 10 +- go.sum | 20 +- integration-tests/go.mod | 12 +- integration-tests/go.sum | 20 +- integration-tests/load/go.mod | 10 +- integration-tests/load/go.sum | 20 +- plugins/plugins.private.yaml | 6 +- plugins/plugins.public.yaml | 6 +- system-tests/lib/go.mod | 10 +- system-tests/lib/go.sum | 20 +- system-tests/tests/go.mod | 10 +- system-tests/tests/go.sum | 20 +- 34 files changed, 879 insertions(+), 268 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 490535635c3..597151d4e45 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -223,7 +223,7 @@ gomodslocalupdate: gomods ## Run gomod-local-update .PHONY: mockery mockery: $(mockery) ## Install mockery. - go install github.com/vektra/mockery/v2@v2.53.0 + GOTOOLCHAIN=go$(shell awk '/^go /{print $$2}' go.mod) go install github.com/vektra/mockery/v2@v2.53.0 .PHONY: codecgen codecgen: $(codecgen) ## Install codecgen diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 14b4be5cc48..f5bb72c9761 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -3157,6 +3157,13 @@ func (b *Metering) ValidateConfig() (err error) { Msg: "requires MeterRecordsEnabled to be true", }) } + if b.MeterRecordsEnabled != nil && *b.MeterRecordsEnabled && (b.NodeID == nil || *b.NodeID == "") { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "NodeID", + Value: "", + Msg: "must be non-empty when MeterRecordsEnabled is true (an empty NodeID collapses per-node snapshot dedup scope DON-wide)", + }) + } return err } diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index bae3d4b1be6..2766aa1570e 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -860,3 +860,65 @@ func durationPtr(d time.Duration) *commonconfig.Duration { cd := *commonconfig.MustNewDuration(d) return &cd } + +func TestMetering_ValidateConfig(t *testing.T) { + testCases := []struct { + name string + config *Metering + expectError bool + errorMsg string + }{ + { + name: "disabled with all nil fields", + config: &Metering{}, + expectError: false, + }, + { + name: "records enabled with non-empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: false, + }, + { + name: "records enabled with nil NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: nil, + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "records enabled with empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new(""), + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "snapshots enabled without records enabled", + config: &Metering{ + MeterSnapshotsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: true, + errorMsg: "requires MeterRecordsEnabled to be true", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.config.ValidateConfig() + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml index 0401c754ad2..2c39f14fd45 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml @@ -40,7 +40,7 @@ name = "workflow" don_family = "test-don-family" don_types = ["workflow"] - override_mode = "all" + override_mode = "each" http_port_range_start = 10100 # even though this DON is not using any capability for chain with ID 2337 we still need it to be connected to it, @@ -57,14 +57,104 @@ image = "postgres:12.0" port = 13000 - [[nodesets.node_specs]] + # override_mode = "each": one node_specs block per node so each node gets a + # distinct [Metering].NodeID (node_id must be unique per node for snapshot + # dedup / cross-node median). Keep count in sync with nodes = 4. + [[nodesets.node_specs]] roles = ["plugin"] [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "core/chainlink.Dockerfile" docker_build_args = { "CL_IS_PROD_BUILD" = "false" } # image = "chainlink-tmp:latest" - user_config_overrides = "" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-1' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-2' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-3' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'wf-zone-a' + NodeID = 'clp-cre-wf-zone-a-4' + """ [[nodesets]] nodes = 4 @@ -72,7 +162,7 @@ don_family = "test-don-family" don_types = ["capabilities"] exposes_remote_capabilities = true - override_mode = "all" + override_mode = "each" http_port_range_start = 10200 # we need to have chain 1337 configured (even if no capability uses it), because we use node addresses on chain 1337 @@ -86,6 +176,8 @@ image = "postgres:12.0" port = 13100 + # override_mode = "each": one node_specs block per node so each node gets a + # distinct [Metering].NodeID. Keep count in sync with nodes = 4. [[nodesets.node_specs]] roles = ["plugin"] [nodesets.node_specs.node] @@ -93,7 +185,94 @@ docker_file = "core/chainlink.Dockerfile" docker_build_args = { "CL_IS_PROD_BUILD" = "false" } # image = "chainlink-tmp:latest" - user_config_overrides = "" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-1' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-2' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-3' + """ + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + # image = "chainlink-tmp:latest" + user_config_overrides = """ + [Telemetry] + Enabled = true + ChipIngressEndpoint = 'chip-ingress:50051' + InsecureConnection = true + + [Metering] + MeterRecordsEnabled = true + MeterSnapshotsEnabled = true + Product = 'cre' + Tenant = 'local-cre' + NumericTenantID = '1' + Environment = 'local' + Zone = 'cap-zone-a' + NodeID = 'clp-cre-cap-zone-a-4' + """ [[nodesets]] nodes = 1 diff --git a/core/scripts/cre/environment/environment/chip_ingress_stack.go b/core/scripts/cre/environment/environment/chip_ingress_stack.go index eaac04360ef..71260340c4e 100644 --- a/core/scripts/cre/environment/environment/chip_ingress_stack.go +++ b/core/scripts/cre/environment/environment/chip_ingress_stack.go @@ -77,10 +77,14 @@ func schemaCommitRefFromGoMod(ctx context.Context, repoRoot, targetModule string // getSchemaSetFromGoMod resolves SchemaSets from chainlink-protos commits pinned in go.mod: // - workflows (chip-cre.json) for CRE/workflow telemetry // - node-platform (chip-schemas.json) for PluginRelayerConfigEmitter / common.v1.ChainPluginConfig +// - metering (chip-cll-meter.json) for durable resource metering (MeterRecord/MeterSnapshot on +// the cll.meter domain); without this, ChIP Ingress rejects those events at pre-publish encode +// time with "Subject 'cll-meter-metering.v1.MeterRecord' not found" and drops them silently. func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, error) { const ( workflowsModule = "github.com/smartcontractkit/chainlink-protos/workflows/go" nodePlatformModule = "github.com/smartcontractkit/chainlink-protos/node-platform" + meteringModule = "github.com/smartcontractkit/chainlink-protos/metering/go" ) repoRoot, err := filepath.Abs(relativePathToRepoRoot) @@ -100,6 +104,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err } framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", nodePlatformModule, npRef, npVer) + meteringRef, meteringVer, err := schemaCommitRefFromGoMod(ctx, repoRoot, meteringModule) + if err != nil { + return nil, err + } + framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", meteringModule, meteringRef, meteringVer) + return []chipingressset.SchemaSet{ { URI: chainlinkProtosGitURI, @@ -113,6 +123,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err SchemaDir: "node-platform", ConfigFile: "chip-schemas.json", }, + { + URI: chainlinkProtosGitURI, + Ref: meteringRef, + SchemaDir: "metering", + ConfigFile: "chip-cll.meter.json", + }, }, nil } @@ -1045,6 +1061,6 @@ func fetchAndRegisterProtosCmd() *cobra.Command { }) }, } - cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "h", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") + cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "u", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") return cmd } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 7ded20dcc73..ade5f438ac3 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.106 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -166,7 +166,7 @@ require ( github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.70.2 // indirect @@ -493,7 +493,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20260423135514-5b1a7565a99c // indirect @@ -508,7 +508,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect @@ -630,7 +630,7 @@ require ( google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.82.0 // indirect + google.golang.org/grpc v1.82.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/guregu/null.v4 v4.0.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 70c38662439..536de3ed278 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -287,8 +287,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1580,12 +1580,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h1:sH546tdm3VaYGaRLnZ6ZYBZiE25vEkMap76Xq/vQWx0= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1626,8 +1626,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2459,8 +2459,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go index 632b2b1fa3b..a36dbb9e597 100644 --- a/core/services/chainlink/config_metering.go +++ b/core/services/chainlink/config_metering.go @@ -22,8 +22,10 @@ func (b *meteringConfig) MeterSnapshotsEnabled() bool { return *b.s.MeterSnapshotsEnabled } -// Product defaults to "unset" (resourcemanager.DefaultMeteringProduct) when unset, -// to catch config errors early. +// Product returns the deployment product identity dimension. The parsed config +// defaults it to "cre" via docs.CoreDefaults so metering is never enabled with +// an empty product dimension; a zero-value toml.Metering that has not been run +// through setDefaults returns "unset" (the nil-pointer fallback below). func (b *meteringConfig) Product() string { if b.s.Product == nil { return "unset" diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go index 42f64260f8c..71eb6eb0da7 100644 --- a/core/services/chainlink/config_metering_test.go +++ b/core/services/chainlink/config_metering_test.go @@ -15,9 +15,11 @@ func TestMeteringConfig(t *testing.T) { mc := meteringConfig{s: toml.Metering{}} assert.False(t, mc.MeterRecordsEnabled()) assert.False(t, mc.MeterSnapshotsEnabled()) - // Product defaults to "cre" so metering can never be enabled with an - // empty product dimension and the syncer identity matches the plugins' - // fallback. + // A zero-value toml.Metering (not run through setDefaults) has a nil + // Product pointer, so Product() returns "unset". The parsed config + // applies a "cre" default via docs.CoreDefaults (covered by the + // LogConfiguration effective-TOML test), so metering is never enabled + // with an empty product dimension. assert.Equal(t, "unset", mc.Product()) assert.Empty(t, mc.Tenant()) assert.Empty(t, mc.NumericTenantID()) diff --git a/core/services/chainlink/mocks/general_config.go b/core/services/chainlink/mocks/general_config.go index 4d72c6bbb57..20e095505b7 100644 --- a/core/services/chainlink/mocks/general_config.go +++ b/core/services/chainlink/mocks/general_config.go @@ -1518,6 +1518,53 @@ func (_c *GeneralConfig_Mercury_Call) RunAndReturn(run func() de.Mercury) *Gener return _c } +// Metering provides a mock function with no fields +func (_m *GeneralConfig) Metering() config.Metering { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Metering") + } + + var r0 config.Metering + if rf, ok := ret.Get(0).(func() config.Metering); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(config.Metering) + } + } + + return r0 +} + +// GeneralConfig_Metering_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Metering' +type GeneralConfig_Metering_Call struct { + *mock.Call +} + +// Metering is a helper method to define mock.On call +func (_e *GeneralConfig_Expecter) Metering() *GeneralConfig_Metering_Call { + return &GeneralConfig_Metering_Call{Call: _e.mock.On("Metering")} +} + +func (_c *GeneralConfig_Metering_Call) Run(run func()) *GeneralConfig_Metering_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *GeneralConfig_Metering_Call) Return(_a0 config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GeneralConfig_Metering_Call) RunAndReturn(run func() config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(run) + return _c +} + // OCR provides a mock function with no fields func (_m *GeneralConfig) OCR() config.OCR { ret := _m.Called() @@ -2604,53 +2651,6 @@ func (_c *GeneralConfig_Telemetry_Call) RunAndReturn(run func() config.Telemetry return _c } -// Metering provides a mock function with no fields -func (_m *GeneralConfig) Metering() config.Metering { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Metering") - } - - var r0 config.Metering - if rf, ok := ret.Get(0).(func() config.Metering); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(config.Metering) - } - } - - return r0 -} - -// GeneralConfig_Metering_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Metering' -type GeneralConfig_Metering_Call struct { - *mock.Call -} - -// Metering is a helper method to define mock.On call -func (_e *GeneralConfig_Expecter) Metering() *GeneralConfig_Metering_Call { - return &GeneralConfig_Metering_Call{Call: _e.mock.On("Metering")} -} - -func (_c *GeneralConfig_Metering_Call) Run(run func()) *GeneralConfig_Metering_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *GeneralConfig_Metering_Call) Return(_a0 config.Metering) *GeneralConfig_Metering_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *GeneralConfig_Metering_Call) RunAndReturn(run func() config.Metering) *GeneralConfig_Metering_Call { - _c.Call.Return(run) - return _c -} - // TelemetryIngress provides a mock function with no fields func (_m *GeneralConfig) TelemetryIngress() config.TelemetryIngress { ret := _m.Called() diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 7d607891451..0b5b7d4539d 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -370,7 +370,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = 'unset' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 55f9cbe7314..e2879ba1cba 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -370,7 +370,7 @@ Prefixes = ['go_'] [Metering] MeterRecordsEnabled = false MeterSnapshotsEnabled = false -Product = 'unset' +Product = 'cre' Tenant = '' NumericTenantID = '' Environment = '' diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index a9467a3cd59..5a618722209 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -126,6 +126,7 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp ` var spec job.WorkflowSpec + // Note: "Get will return sql.ErrNoRows like row.Scan would" - sqlx@v1.4.0 err := orm.ds.GetContext(ctx, &spec, query, id) if err != nil { return nil, err diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 85c160cbca3..a7e9d4c41b4 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -2,6 +2,7 @@ package v2 import ( "context" + "database/sql" "encoding/hex" "errors" "fmt" @@ -301,9 +302,6 @@ func WithModuleEngineVersion(v string) func(*eventHandler) { type WorkflowArtifactsStore interface { FetchWorkflowArtifacts(ctx context.Context, workflowID, binaryIdentifier, configIdentifier string) ([]byte, []byte, error) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) - // GetWorkflowSpecList returns the persisted workflow specs. It backs the - // metering snapshot path (Meterable.GetUtilization), which enumerates the - // durable specs rather than running engines. GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error @@ -358,11 +356,7 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - // resourceManager defaults to nil (metering off); it is set only via - // WithResourceManager. A nil manager keeps every emission site guarded - // and avoids registering duplicate OTel metering instruments. - // Default identity carries only the service-level constants; the coarse - // deployment/node/DON dimensions are filled in by WithIdentity in cre.go. + tracer: noop.NewTracerProvider().Tracer(""), // default; can override in WithDebugMode meterIdentity: resourcemanager.ResourceIdentity{ Service: meterService, ResourcePool: meterResourcePool, @@ -391,15 +385,12 @@ func (h *eventHandler) start(ctx context.Context) error { if h.moduleLRU != nil { h.moduleLRU.Start() } - // The handler is the single owner of its ResourceManager: it starts the RM - // (which owns the snapshot tick) and registers itself as the Meterable that - // is snapshotted. The RM is a no-op when metering is disabled, so this is - // safe regardless of the gate. + if h.resourceManager != nil { if err := h.resourceManager.Start(ctx); err != nil { return fmt.Errorf("failed to start resource manager: %w", err) } - // Register returns a func that unregisters. + // Register returns a func that unregisters. h.rmUnregister = h.resourceManager.Register(h) h.resolveWorkflowDonID() } @@ -575,21 +566,22 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { wfID := payload.WorkflowID.Hex() - // Get workflow spec from database to get owner and name info for organization lookup - // Alternative: wire through workflowOwner into the Event, but that requires a lot more surgery - spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID) + // Get workflow spec from database to get owner and name info for organization lookup, + // and to check if the spec exists to determine if a MeterRecord is warranted. var wfOwner, wfName, orgID string - if err != nil { - // Workflow spec not found, proceed with deletion but without event metadata - h.lggr.Warnw("Workflow spec not found during deletion, proceeding without org info", "workflowID", wfID, "error", err) - } else { + specExisted := false + if spec, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID); gerr == nil && spec != nil { + specExisted = true wfOwner = spec.WorkflowOwner wfName = spec.WorkflowName - if wfOwner != "" { - orgID, err = h.fetchOrganizationID(ctx, wfOwner) - if err != nil { - h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", err) - } + } else if gerr != nil && !errors.Is(gerr, sql.ErrNoRows) { + h.lggr.Warnw("failed to read workflow spec during deletion, proceeding without metadata", "workflowID", wfID, "error", gerr) + } + if wfOwner != "" { + if oerr, ferr := h.fetchOrganizationID(ctx, wfOwner); ferr != nil { + h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", ferr) + } else { + orgID = oerr } } ctx = contexts.WithCRE(ctx, contexts.CRE{Org: orgID, Owner: wfOwner, Workflow: wfID}) @@ -613,7 +605,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { } }() - if herr = h.workflowDeletedEvent(ctx, payload, WorkflowDeleted); herr != nil { + if herr = h.workflowDeletedEvent(ctx, payload, WorkflowDeleted, wfOwner, specExisted); herr != nil { if errors.Is(herr, ErrDrainInProgress) { logCustMsg(ctx, cma, fmt.Sprintf("workflow deletion deferred: %v", herr), h.lggr) } else { @@ -637,7 +629,7 @@ func (h *eventHandler) workflowActivatedEvent( ) error { // Convert WorkflowActivatedEvent to WorkflowRegisteredEvent since they have identical fields registeredPayload := WorkflowRegisteredEvent(payload) - return h.workflowRegisteredEvent(ctx, registeredPayload, WorkflowActivated) + return h.workflowRegisteredEvent(ctx, registeredPayload) } // workflowRegisteredEvent handles the WorkflowRegisteredEvent event type. @@ -645,13 +637,9 @@ func (h *eventHandler) workflowActivatedEvent( // workflowRegisteredEvent proceeds in two phases: // - phase 1 synchronizes the database state // - phase 2 synchronizes the state of the engine registry. -// originatingEvent names the event that triggered this call (e.g. WorkflowActivated -// when delegated from workflowActivatedEvent) and identifies the originating intent -// in emitted meter records. func (h *eventHandler) workflowRegisteredEvent( ctx context.Context, payload WorkflowRegisteredEvent, - originatingEvent WorkflowRegistryEventName, ) error { ctx, span := h.tracer.Start(ctx, "workflow_registered", trace.WithAttributes( @@ -669,17 +657,17 @@ func (h *eventHandler) workflowRegisteredEvent( // - existing registration that has been updated with a new status spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, payload.WorkflowID.Hex()) switch { - case err != nil: + case errors.Is(err, sql.ErrNoRows): newSpec, innerErr := h.createWorkflowSpec(ctx, payload) if innerErr != nil { return innerErr } - // A new spec's artifacts were persisted for the first time: the durable - // workflow_specs_v2 resource gained one unit, so emit a +1 delta. h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) spec = newSpec + case err != nil: + return fmt.Errorf("failed to get workflow spec: %w", err) case spec.WorkflowID != payload.WorkflowID.Hex(): newSpec, innerErr := h.createWorkflowSpec(ctx, payload) if innerErr != nil { @@ -944,21 +932,31 @@ func (h *eventHandler) createEngineModule( } // workflowPausedEvent handles the WorkflowPausedEvent event type. This method must remain idempotent. +// +// Pause emits no metering delta (by design): it removes artifacts and stops the +// engine, but the durable spec's release is realized by its absence from +// subsequent snapshots, not a -1 delta. Activate-after-pause re-runs the create +// path and emits a +1 with the register event_id (same on-chain CreatedAt), so a +// within-bucket pause→activate merges with the original register; cumulative-delta +// drift is corrected by snapshot level reconciliation. func (h *eventHandler) workflowPausedEvent( ctx context.Context, payload WorkflowPausedEvent, ) error { - return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}, WorkflowPaused) + return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}, WorkflowPaused, "", false) } // workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. // originatingEvent names the event that triggered this call (e.g. WorkflowPaused -// when delegated from workflowPausedEvent) and identifies the originating intent -// in emitted meter records. +// when delegated from workflowPausedEvent). deleteOwner/specExisted are supplied +// by the caller (Handle pre-reads the spec once for labels and to decide whether +// a -1 is warranted); pause passes ""/false since it emits no delta. func (h *eventHandler) workflowDeletedEvent( ctx context.Context, payload WorkflowDeletedEvent, originatingEvent WorkflowRegistryEventName, + deleteOwner string, + specExisted bool, ) error { // The order in the handler is slightly different to the order in `tryEngineCleanup`. // This is because the engine requires its corresponding DB record to be present to be successfully @@ -966,16 +964,6 @@ func (h *eventHandler) workflowDeletedEvent( // At the same time, popping the engine should occur last to allow deletes to be retried if any of the // prior steps fail. workflowID := payload.WorkflowID.Hex() - // Capture the stored owner before deletion so a real-delete -1 delta can - // resolve org attribution from durable state. Only needed for actual - // deletions (pause emits no delta); fail-open, an empty owner yields an - // empty org. - var deleteOwner string - if originatingEvent == WorkflowDeleted { - if existing, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, workflowID); gerr == nil && existing != nil { - deleteOwner = existing.WorkflowOwner - } - } e, ok := h.engineRegistry.Get(payload.WorkflowID) var drainable DrainableService var isDrainable bool @@ -1007,15 +995,13 @@ func (h *eventHandler) workflowDeletedEvent( if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } - // A real delete removed the persisted spec, so emit a -1 delta. Pause is - // delegated here too (originatingEvent == WorkflowPaused); its release is - // realized by the spec's absence from subsequent snapshots, so pause emits - // no delta. - if originatingEvent == WorkflowDeleted { - // WorkflowDeletedEvent carries only the workflow ID (no on-chain - // CreatedAt/block), so the delete event_id is derived from the - // DON-consistent workflowID alone. See the emitSpecDelta call for the - // re-delete-cycle limitation noted in the metering redesign. + // A real delete that removed a previously-persisted spec emits a -1 delta. + // Gating on specExisted prevents a redelivered delete (spec already removed, + // and DeleteWorkflowArtifacts maps ErrNoRows→nil) from emitting a second -1. + // Pause is delegated here too (originatingEvent == WorkflowPaused); it passes + // specExisted=false to avoid emitting a delta + if originatingEvent == WorkflowDeleted && specExisted { + // eventID must be consistent across DONs h.emitSpecDelta(ctx, -1, workflowID, deleteOwner, resourcemanager.EventID("workflow-spec-delete", workflowID)) } @@ -1103,19 +1089,22 @@ func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { return h.baseIdentity() } +// GetWorkflowSpecList allows meter deltas to ignore activate-then-pause and +// pause-then-activate events. +func (h *eventHandler) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.workflowArtifactsStore.GetWorkflowSpecList(ctx) +} + // GetUtilization implements resourcemanager.Meterable: it returns one -// SnapshotEntry per persisted workflow_specs_v2 spec, each at level 1, so the -// periodic absolute-state snapshot agrees with the +1/-1 deltas emitted on -// artifact-persistence transitions. The durable resource is the stored spec, -// not the running engine, so this enumerates persisted specs (via the store's -// list) rather than the engine registry — that way paused-but-stored specs and +// SnapshotEntry per persisted workflow_specs_v2 spec. +// The durable resource is the stored spec, NOT the running engine. +// Why: paused-but-stored specs and // specs whose engine failed to start are still accounted for, and a delete is // reflected by the spec's absence from the next snapshot. // -// resource_id is the workflow_id; the organization is resolved from the spec's -// stored owner through the caching resolver, which serves from memory so this -// tick stays effectively no-network. Fail-open: a store error yields no entries -// for this tick. +// resource_id is the workflow_id; the organization is resolved fail-open from +// the spec's stored owner through the org resolver. Fail-open: a store error +// yields no entries for this tick. func (h *eventHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { specs, err := h.workflowArtifactsStore.GetWorkflowSpecList(ctx) if err != nil { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index 6c4ea25f067..8c052e8ba17 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "math/big" + "strconv" "sync" "testing" "time" @@ -17,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/custmsg" commonlogger "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" @@ -174,7 +176,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: wfID, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }, WorkflowRegistered) + }) require.NoError(t, err) records := emitter.Records() @@ -198,8 +200,8 @@ func Test_meterRecords(t *testing.T) { WorkflowName: "wf-name", CreatedAt: 123, } - require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) - require.NoError(t, h.workflowRegisteredEvent(t.Context(), event, WorkflowRegistered)) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) records := emitter.Records() require.Len(t, records, 2) @@ -232,7 +234,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: wfID, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }, WorkflowRegistered) + }) require.NoError(t, err) assert.Empty(t, emitter.Records()) @@ -269,7 +271,7 @@ func Test_meterRecords(t *testing.T) { }, }, newMeteringResourceManager(t, true, emitter)) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) records := emitter.Records() require.Len(t, records, 1) @@ -287,7 +289,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: types.WorkflowID{6}, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }, WorkflowRegistered) + }) require.ErrorIs(t, err, assert.AnError) assert.Empty(t, emitter.Records()) }) @@ -305,7 +307,7 @@ func Test_meterRecords(t *testing.T) { deleteErr: assert.AnError, }, newMeteringResourceManager(t, true, emitter)) - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true) require.ErrorIs(t, err, assert.AnError) assert.Empty(t, emitter.Records()) }) @@ -326,12 +328,12 @@ func Test_meterRecords(t *testing.T) { drainable.activeExecutions.Store(1) require.NoError(t, h.engineRegistry.Add(wfID, "test-source", drainable)) - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true) require.ErrorIs(t, err, ErrDrainInProgress) assert.Empty(t, emitter.Records()) drainable.activeExecutions.Store(0) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) records := emitter.Records() require.Len(t, records, 1) @@ -356,8 +358,8 @@ func Test_meterRecords(t *testing.T) { WorkflowID: wfID, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }, WorkflowRegistered)) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted)) + })) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) assert.Empty(t, emitter.Records()) }) @@ -371,7 +373,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: types.WorkflowID{10}, WorkflowOwner: wfOwner, WorkflowName: "wf-name", - }, WorkflowRegistered)) + })) assert.Empty(t, emitter.Records()) }) @@ -425,3 +427,204 @@ func Test_meterRecords(t *testing.T) { require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second persisted spec") }) } + +// fakeOrgResolver is a no-network OrgResolver that maps workflow owners (hex) +// to organization IDs. It asserts that metering org resolution is exercised +// (previously org was entirely untested). +type fakeOrgResolver struct { + orgs map[string]string +} + +func (f *fakeOrgResolver) Get(_ context.Context, owner string) (string, error) { + if org, ok := f.orgs[owner]; ok { + return org, nil + } + return "", errors.New("owner not found") +} + +func (f *fakeOrgResolver) Start(context.Context) error { return nil } +func (f *fakeOrgResolver) Close() error { return nil } +func (f *fakeOrgResolver) HealthReport() map[string]error { return nil } +func (f *fakeOrgResolver) Ready() error { return nil } +func (f *fakeOrgResolver) Name() string { return "fake-org-resolver" } + +// newMeteringTestHandlerWithOrg is newMeteringTestHandler plus a wired +// OrgResolver, so tests can assert OrgId on emitted records. +func newMeteringTestHandlerWithOrg(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager, org orgresolver.OrgResolver) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + &confidentialrelay.ExecutionHandlers{}, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithResourceManager(rm), + WithEngineFactoryFn(mockEngineFactory), + WithOrgResolver(org), + ) + require.NoError(t, err) + return h +} + +// redeliveredDeleteSpecs returns a fresh stub seeded with a stored spec and the +// owner the metering tests use, so a delete + redelivered-delete pair can be +// driven through Handle (which pre-reads the spec itself). +func redeliveredDeleteStore() *stubWorkflowArtifactsStore { + return &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{5}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + } +} + +// a redelivered delete (after the spec was already removed) must NOT emit a +// second -1. DeleteWorkflowArtifacts maps ErrNoRows→nil, so without gating on +// the pre-read the redelivery would double-emit. +func Test_meterRecords_RedeliveredDeleteEmitsExactlyOneDelta(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{5} + emitter := &recordingEmitter{} + store := redeliveredDeleteStore() + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + deleteEvent := Event{Name: WorkflowDeleted, Data: WorkflowDeletedEvent{WorkflowID: wfID}} + require.NoError(t, h.Handle(t.Context(), deleteEvent)) + require.NoError(t, h.Handle(t.Context(), deleteEvent)) // redelivery + + records := emitter.Records() + require.Len(t, records, 1, "redelivered delete must not emit a second -1") + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) +} + +// activate-after-pause re-runs the create path and emits a +1 whose +// event_id equals the original register event_id (same on-chain CreatedAt). +// Pause itself emits no delta; within-bucket pause→activate therefore merges +// with the original register, with cumulative-delta drift corrected by snapshots. +func Test_meterRecords_ActivateAfterPauseEmitsRegisterEventID(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + // The stub does not persist on upsert, so register and activate each take + // the new-spec path; pause deletes (a no-op here) and emits nothing. + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{42} + createdAt := uint64(123) + wantEventID := resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10)) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + + records := emitter.Records() + require.Len(t, records, 2, "register and activate each emit one +1; pause emits none") + requireSpecDelta(t, records[0], "1", wfID.Hex(), wantEventID) + requireSpecDelta(t, records[1], "1", wfID.Hex(), wantEventID) + assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId(), + "activate-after-pause must reuse the register event_id (same on-chain CreatedAt)") +} + +// OrgId is resolved from the workflow owner and stamped on emitted records. +func Test_meterRecords_OrgIdResolvedOnRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + orgR := &fakeOrgResolver{orgs: map[string]string{"aabbccdd": "org-42"}} + h := newMeteringTestHandlerWithOrg(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter), orgR) + + wfID := types.WorkflowID{50} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + + records := emitter.Records() + require.Len(t, records, 1) + require.Len(t, records[0].Utilizations, 1) + assert.Equal(t, "org-42", records[0].Utilizations[0].OrgId, "OrgId must be resolved and stamped on the record") +} + +// don_id is folded into the metering identity once resolved and stamped on both +// emitted records and snapshot entries. +func Test_meterRecords_DonIDOnRecordAndSnapshot(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + wfID := types.WorkflowID{60} + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{{WorkflowID: wfID.Hex(), WorkflowOwner: "aabbccdd"}}, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + // Simulate a resolved workflow DON id (as resolveWorkflowDonID would store). + resolvedDon := "7" + h.resolvedDonID.Store(&resolvedDon) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + records := emitter.Records() + require.Len(t, records, 1) + require.NotNil(t, records[0].Identity.GetDon()) + assert.Equal(t, "7", records[0].Identity.GetDon().GetDonId(), "record identity must carry resolved don_id") + + entries := h.GetUtilization(t.Context()) + require.Len(t, entries, 1) + require.NotNil(t, entries[0].Identity.Don) + assert.Equal(t, "7", entries[0].Identity.Don.DonID, "snapshot identity must carry resolved don_id") +} + +// a transient DB error on GetWorkflowSpec (not a missing row) must surface +// an error and emit no +1, so the event is retried instead of creating a +// spurious spec + delta. +func Test_meterRecords_TransientDBErrorOnGetSpecEmitsNoDelta(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{getSpecErr: errors.New("db connection lost")}, + newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{70}, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + }) + require.Error(t, err) + assert.Empty(t, emitter.Records(), "transient DB error must not take the new-spec path or emit a +1") +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index e72c708bf1d..79733b20ec8 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -2,6 +2,7 @@ package v2 import ( "context" + "database/sql" "encoding/base64" "encoding/hex" "errors" @@ -245,7 +246,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedURLParameter := "?auth=abc123" defaultValidationFn := func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, _ *mockFetcher) { - err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err := h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) // Verify the record is updated in the database @@ -387,7 +388,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { validationFn: func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL string, configURL string, ) { - err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err := h.workflowRegisteredEvent(ctx, event) require.Error(t, err) require.ErrorIs(t, err, assert.AnError) }, @@ -425,7 +426,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { me := &mockEngine{} err := h.engineRegistry.Add(wfID, event.Source, me) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) }, }, @@ -464,7 +465,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { oldWfIDBytes := [32]byte{0, 1, 2, 3, 5} err := h.engineRegistry.Add(oldWfIDBytes, event.Source, me) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) engineInRegistry, ok := h.engineRegistry.Get(wfID) assert.True(t, ok) @@ -503,7 +504,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { validationFn: func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL string, configURL string, ) { - err := h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err := h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) // Verify the record is updated in the database @@ -566,7 +567,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { _, err := s.UpsertWorkflowSpec(ctx, entry) require.NoError(t, err) - err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) // Verify the record is updated in the database @@ -792,7 +793,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { } ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) assert.Eventually(t, confidential.ran.Load, 10*time.Second, time.Millisecond) @@ -881,7 +882,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { } ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, event, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, event) require.NoError(t, err) assert.Eventually(t, action.ran.Load, 10*time.Second, time.Millisecond) @@ -1111,7 +1112,7 @@ func Test_workflowDeletedHandler(t *testing.T) { ) require.NoError(t, err) ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, active, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, active) require.NoError(t, err) // Verify the record is updated in the database @@ -1130,7 +1131,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) require.NoError(t, err) // Verify the record is deleted in the database @@ -1185,7 +1186,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) require.NoError(t, err) // Verify the record is deleted in the database @@ -1266,7 +1267,7 @@ func Test_workflowDeletedHandler(t *testing.T) { ) require.NoError(t, err) ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) - err = h.workflowRegisteredEvent(ctx, active, WorkflowRegistered) + err = h.workflowRegisteredEvent(ctx, active) require.NoError(t, err) // Verify the record is updated in the database @@ -1285,7 +1286,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted) + err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) require.Error(t, err, failWith) // Verify the record is still in the DB @@ -1304,6 +1305,7 @@ type stubWorkflowArtifactsStore struct { upsertErr error deleteErr error listErr error + getSpecErr error deleteCalls atomic.Int32 } @@ -1312,8 +1314,11 @@ func (s *stubWorkflowArtifactsStore) FetchWorkflowArtifacts(context.Context, str } func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (*job.WorkflowSpec, error) { + if s.getSpecErr != nil { + return nil, s.getSpecErr + } if s.spec == nil { - return nil, errors.New("not found") + return nil, sql.ErrNoRows } return s.spec, nil } @@ -1334,7 +1339,13 @@ func (s *stubWorkflowArtifactsStore) GetWorkflowSpecList(context.Context) ([]*jo func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) error { s.deleteCalls.Add(1) - return s.deleteErr + if s.deleteErr != nil { + return s.deleteErr + } + // Mimic the real store: a successful delete removes the spec, so a + // redelivered delete observes no row (sql.ErrNoRows) on the next read. + s.spec = nil + return nil } func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifactsBatch(context.Context, []string) error { @@ -1357,7 +1368,7 @@ func Test_workflowDeletedEvent_DrainInProgress(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted, "", false) require.Error(t, err) require.ErrorIs(t, err, ErrDrainInProgress) assert.Equal(t, int32(1), drainable.drainCalls.Load()) @@ -1383,7 +1394,7 @@ func Test_workflowDeletedEvent_IgnoresErrAlreadyStopped(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted, "", false) require.NoError(t, err) assert.Equal(t, int32(1), drainable.closeCalls.Load()) assert.Equal(t, int32(1), artifactStore.deleteCalls.Load()) @@ -1421,7 +1432,7 @@ func Test_workflowRegisteredEvent_DrainingEngineNotTreatedAsHealthy(t *testing.T err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ Status: WorkflowStatusActive, WorkflowID: workflowID, - }, WorkflowRegistered) + }) require.Error(t, err) require.Contains(t, err.Error(), "could not clean up old engine") assert.Equal(t, int32(1), drainable.closeCalls.Load()) @@ -1486,7 +1497,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold }) // Mock ORM responses - mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, errors.New("not found")) + mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, sql.ErrNoRows) mockORM.EXPECT().UpsertWorkflowSpec(mock.Anything, mock.AnythingOfType("*job.WorkflowSpec")).Return(int64(1), nil) // Set up handler diff --git a/core/services/workflows/syncer/v2/helpers.go b/core/services/workflows/syncer/v2/helpers.go index e8be21d6d7f..2ebe70f0cb5 100644 --- a/core/services/workflows/syncer/v2/helpers.go +++ b/core/services/workflows/syncer/v2/helpers.go @@ -11,6 +11,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" ) @@ -78,6 +79,10 @@ func (m *testEvtHandler) EmitActivationAbandoned(context.Context, Event, eventsv return nil } +func (m *testEvtHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + return nil, nil +} + func (m *testEvtHandler) ClearEvents() { m.mux.Lock() defer m.mux.Unlock() diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index deb591b6f8c..06108f0feb0 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -26,8 +26,10 @@ import ( "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" "github.com/smartcontractkit/chainlink-evm/pkg/config" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/versioning" + wftypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" ) const name = "WorkflowRegistrySyncer" @@ -137,6 +139,7 @@ type evtHandler interface { Handle(ctx context.Context, event Event) error EmitActivationAbandoned(ctx context.Context, event Event, reason eventsv2.ActivationAbandonReason, activationErr error, retryCount int32) error + GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) } type donNotifier interface { @@ -803,6 +806,14 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) // Process each source independently to isolate failures totalWorkflowsFetched := 0 reconcileReport := newReconcileReport() + // allMetadataIDs is the union of every source's (post-shard) + // workflow IDs this tick; it is the source of truth for the + // downtime-orphan sweep. sourceFetchFailed stays false only when + // every source fetched successfully, so the sweep never runs on + // incomplete metadata (it would risk deleting specs whose workflow + // belongs to a source that failed to fetch). + allMetadataIDs := make(map[string]struct{}) + sourceFetchFailed := false for _, source := range w.workflowSources { sourceName := source.Name() @@ -823,6 +834,7 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) w.metrics.recordSourceFetch(ctx, sourceName, len(workflows), duration, fetchErr) if fetchErr != nil { + sourceFetchFailed = true w.lggr.Errorw("Failed to fetch from source, skipping reconciliation for this source", "source", sourceName, "error", fetchErr, "durationMs", duration.Milliseconds()) // KEY: Skip this source entirely - no events generated, no deletions @@ -839,6 +851,7 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) if w.shardingEnabled { filteredWorkflowsMetadata, err = w.filterWorkflowsByShard(ctx, workflows) if err != nil { + sourceFetchFailed = true w.lggr.Errorw("failed to filter workflows by shard", "err", err, "source", sourceName) @@ -852,6 +865,10 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + for _, wfMeta := range filteredWorkflowsMetadata { + allMetadataIDs[wfMeta.WorkflowID.Hex()] = struct{}{} + } + // Generate events only for this source's engines (using sourceIdentifier for engine registry lookups) events, genErr := w.generateReconciliationEvents(ctx, pendingEvents, filteredWorkflowsMetadata, head, sourceIdentifier) if genErr != nil { @@ -962,6 +979,17 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + // Downtime-orphan sweep: after every source has been reconciled (and + // only when all fetched successfully, so the metadata union is + // complete), delete persisted specs whose workflowID is absent from + // all sources. This catches workflows deleted on-chain while this + // node was down — their spec rows linger with no engine and no + // source event. Dispatched through Handle, they take the existing + // delete path (artifact cleanup + -1 delta) with no new emission code. + if !sourceFetchFailed && len(w.workflowSources) > 0 { + w.reconcileOrphanedSpecs(ctx, allMetadataIDs) + } + w.metrics.recordFetchedWorkflows(ctx, totalWorkflowsFetched) w.lggr.Debugw("reconciled events", "report", reconcileReport) @@ -983,6 +1011,48 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) } } +// reconcileOrphanedSpecs dispatches a WorkflowDeleted event for every persisted +// workflow spec whose workflowID is absent from liveWorkflowIDs (the union of +// every source's metadata this tick). It uses EXISTING state — the persisted +// spec rows are this node's record of "workflows it believes exist" — and the +// registry metadata is the source of truth. Each orphan flows through the +// existing handler delete path, producing the -1 delta and artifact cleanup with +// zero new emission code. Paused workflows remain in metadata (status Paused), +// so they are present in liveWorkflowIDs and are never swept. Fail-open: a +// listing error or an unparseable id skips the sweep / that row. +func (w *workflowRegistry) reconcileOrphanedSpecs(ctx context.Context, liveWorkflowIDs map[string]struct{}) { + specs, err := w.handler.GetWorkflowSpecList(ctx) + if err != nil { + w.lggr.Warnw("failed to list persisted workflow specs for orphan reconciliation; skipping sweep", "err", err) + return + } + for _, spec := range specs { + if spec == nil || spec.WorkflowID == "" { + continue + } + if _, live := liveWorkflowIDs[spec.WorkflowID]; live { + continue + } + wfIDBytes, derr := hex.DecodeString(spec.WorkflowID) + if derr != nil || len(wfIDBytes) != len(wftypes.WorkflowID{}) { + w.lggr.Warnw("orphan sweep: skipping unparseable persisted workflow_id", "workflowID", spec.WorkflowID, "err", derr) + continue + } + var wfID wftypes.WorkflowID + copy(wfID[:], wfIDBytes) + w.lggr.Debugw("orphaned workflow spec: persisted locally but absent from registry metadata; dispatching delete", + "workflowID", spec.WorkflowID, "owner", spec.WorkflowOwner) + evt := Event{ + Name: WorkflowDeleted, + Data: WorkflowDeletedEvent{WorkflowID: wfID}, + Head: Head{}, + } + if herr := w.handler.Handle(ctx, evt); herr != nil { + w.lggr.Warnw("failed to handle orphaned workflow spec delete", "workflowID", spec.WorkflowID, "err", herr) + } + } +} + // getTicker returns the ticker that the workflowRegistry will use to poll for events. If the ticker // is nil, then a default ticker is returned. func (w *workflowRegistry) getTicker(d time.Duration) <-chan time.Time { diff --git a/core/services/workflows/syncer/v2/workflow_registry_test.go b/core/services/workflows/syncer/v2/workflow_registry_test.go index 6c2b9e7b207..95da83875b6 100644 --- a/core/services/workflows/syncer/v2/workflow_registry_test.go +++ b/core/services/workflows/syncer/v2/workflow_registry_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" commonCap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -20,8 +21,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" + eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" wfTypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" @@ -1839,3 +1842,61 @@ func TestWorkflowRegistry_getTicker_WithTickerOverride(t *testing.T) { tickerCh := wr.getTicker(10 * time.Second) require.Equal(t, (<-chan time.Time)(customCh), tickerCh) } + +// orphanSweepFakeHandler is a minimal evtHandler that returns a fixed spec list +// and records every Handle call, so reconcileOrphanedSpecs can be unit-tested +// in isolation. +type orphanSweepFakeHandler struct { + mu sync.Mutex + specs []*job.WorkflowSpec + handled []Event +} + +func (h *orphanSweepFakeHandler) Close() error { return nil } +func (h *orphanSweepFakeHandler) Start(context.Context) error { return nil } +func (h *orphanSweepFakeHandler) Handle(_ context.Context, event Event) error { + h.mu.Lock() + defer h.mu.Unlock() + h.handled = append(h.handled, event) + return nil +} +func (h *orphanSweepFakeHandler) EmitActivationAbandoned(context.Context, Event, eventsv2.ActivationAbandonReason, error, int32) error { + return nil +} +func (h *orphanSweepFakeHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { + return h.specs, nil +} +func (h *orphanSweepFakeHandler) Handled() []Event { + h.mu.Lock() + defer h.mu.Unlock() + cp := make([]Event, len(h.handled)) + copy(cp, h.handled) + return cp +} + +// a persisted spec whose workflowID is absent from the registry metadata +// (the union of all sources) is a downtime orphan — it is dispatched through +// the existing delete path. Specs still present in metadata (including paused +// workflows, which remain in metadata) are left alone. +func Test_reconcileOrphanedSpecs_DispatchesDeleteForOrphansOnly(t *testing.T) { + t.Parallel() + liveID := wfTypes.WorkflowID{1} + orphanID := wfTypes.WorkflowID{2} + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, + }} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h} + + // liveWorkflowIDs is the union of all sources' metadata this tick; the + // live workflow (present in metadata) is retained, the orphan is swept. + live := map[string]struct{}{liveID.Hex(): {}} + w.reconcileOrphanedSpecs(t.Context(), live) + + handled := h.Handled() + require.Len(t, handled, 1, "exactly the orphan is swept") + assert.Equal(t, WorkflowDeleted, handled[0].Name) + del, ok := handled[0].Data.(WorkflowDeletedEvent) + require.True(t, ok) + assert.Equal(t, orphanID, del.WorkflowID, "only the spec absent from metadata is deleted") +} diff --git a/deployment/go.mod b/deployment/go.mod index b39e7d8e17d..c6ffbda26f5 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 @@ -83,7 +83,7 @@ require ( golang.org/x/mod v0.37.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/yaml.v3 v3.0.1 @@ -164,7 +164,7 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -427,7 +427,7 @@ require ( github.com/smartcontractkit/chainlink-automation v0.8.1 // indirect github.com/smartcontractkit/chainlink-canton v0.0.0-20260615233851-4e78e7c23a58 // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20260423135514-5b1a7565a99c // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260709082627-78ab5315e367 // indirect @@ -441,7 +441,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/deployment/go.sum b/deployment/go.sum index f3620bf625d..08618d3253d 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -282,8 +282,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1390,12 +1390,12 @@ github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1436,8 +1436,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2259,8 +2259,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index dffb01580fc..d6c03c02a0d 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2658,6 +2658,7 @@ Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. NodeID = '' # Default ``` NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). +Required (non-empty) when `MeterRecordsEnabled = true`. ## CRE.Streams ```toml diff --git a/go.mod b/go.mod index 3e6a887a0bc..48204ee281a 100644 --- a/go.mod +++ b/go.mod @@ -85,9 +85,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -100,7 +100,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260623200841-e0322b819f62 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd @@ -144,7 +144,7 @@ require ( golang.org/x/term v0.45.0 golang.org/x/time v0.15.0 gonum.org/v1/gonum v0.17.0 - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -189,7 +189,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/btcutil v1.1.6 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/sonic v1.12.3 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index ff97e988c77..ea5f26922bc 100644 --- a/go.sum +++ b/go.sum @@ -194,8 +194,8 @@ github.com/buger/jsonparser v1.2.0 h1:4EFcvK1kD4jyj6YqNK6skK6w+y7FHHBR+XBCtxwu/6 github.com/buger/jsonparser v1.2.0/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -1159,12 +1159,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h1:sH546tdm3VaYGaRLnZ6ZYBZiE25vEkMap76Xq/vQWx0= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 h1:Oe2G2RQTgfy9nSIW699IdRK+V7Z6euVsaueFnMNvTEc= @@ -1201,8 +1201,8 @@ github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546- github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= @@ -1948,8 +1948,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index a5f173b37a4..dffaaa38415 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -33,7 +33,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 @@ -56,11 +56,14 @@ require ( golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a golang.org/x/sync v0.22.0 golang.org/x/text v0.40.0 - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 gopkg.in/guregu/null.v4 v4.0.0 ) -require github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect +require ( + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect +) require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect @@ -134,7 +137,6 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -406,7 +408,7 @@ require ( github.com/smartcontractkit/chainlink-canton v0.0.0-20260615233851-4e78e7c23a58 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 59a7e2512ab..6dc0e68134c 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -270,8 +270,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1377,12 +1377,12 @@ github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1423,8 +1423,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2239,8 +2239,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 821221b306c..408bc49d2f0 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.6-0.20260708113039-95f97b2d25e9 @@ -40,6 +40,7 @@ require ( ) require ( + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect @@ -131,7 +132,6 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -490,7 +490,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b // indirect @@ -509,7 +509,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect; indirect develop + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect; indirect develop github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/rmn/v1.6/go v0.0.0-20250131130834-15e0d4cde2a6 // indirect github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 // indirect @@ -650,7 +650,7 @@ require ( google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.82.0 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/guregu/null.v4 v4.0.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 8260b79d0c0..b543b71d1db 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -325,8 +325,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1637,12 +1637,12 @@ github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194 h1:QxZkbKtQyPtVLYP4eMwc+VbXY7M5ve1deSiLZ2pOA+Y= github.com/smartcontractkit/chainlink-ccv/deployment v0.0.2-0.20260616151800-9a3a31c4e194/go.mod h1:bNMFRxwWdgVFdSsFZRmsUUPoBUncU3RM765K99svIKM= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1683,8 +1683,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2630,8 +2630,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 24d0bfa8eff..e6151a3d791 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -9,7 +9,7 @@ defaults: plugins: cron: - moduleURI: "github.com/smartcontractkit/capabilities/cron" - gitRef: "792fd475a9c294b61fa90b9dd23388da2827f093" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." flags: "-tags timetzdata" readcontract: @@ -31,11 +31,11 @@ plugins: installPath: "." httptrigger: - moduleURI: "github.com/smartcontractkit/capabilities/http_trigger" - gitRef: "2cf965790a71171e1336f6d1d4f5162f2bd39bd1" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." evm: - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/evm" - gitRef: "d7162a2b15aea2ce861052ebaa2f5ca716a9b60a" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." solana: - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/solana" diff --git a/plugins/plugins.public.yaml b/plugins/plugins.public.yaml index 027a4092aa7..317426b03a6 100644 --- a/plugins/plugins.public.yaml +++ b/plugins/plugins.public.yaml @@ -71,7 +71,7 @@ plugins: capability-cron: - moduleURI: "github.com/smartcontractkit/capabilities/cron" - gitRef: "792fd475a9c294b61fa90b9dd23388da2827f093" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." flags: "-tags timetzdata" @@ -98,12 +98,12 @@ plugins: capability-httptrigger: - moduleURI: "github.com/smartcontractkit/capabilities/http_trigger" - gitRef: "2cf965790a71171e1336f6d1d4f5162f2bd39bd1" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." capability-evm: - moduleURI: "github.com/smartcontractkit/capabilities/chain_capabilities/evm" - gitRef: "d7162a2b15aea2ce861052ebaa2f5ca716a9b60a" + gitRef: "62d191a19372b65ce34539220702d92067d8aa3f" installPath: "." capability-solana: diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 1cc500dbfe1..63d50b90dd8 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.106 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260708114855-e953eeb028a7 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 @@ -61,7 +61,7 @@ require ( go.uber.org/ratelimit v0.3.1 go.uber.org/zap v1.28.0 golang.org/x/sync v0.22.0 - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 @@ -151,7 +151,7 @@ require ( github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.70.2 // indirect @@ -459,7 +459,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect @@ -474,7 +474,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 07fb9d503a0..505f658a206 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -282,8 +282,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1551,12 +1551,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h1:sH546tdm3VaYGaRLnZ6ZYBZiE25vEkMap76Xq/vQWx0= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1597,8 +1597,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2419,8 +2419,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index e0624853060..c9bd633e805 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -62,7 +62,7 @@ require ( github.com/rs/zerolog v1.35.1 github.com/smartcontractkit/chain-selectors v1.0.106 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -121,6 +121,7 @@ require ( github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect github.com/beevik/ntp v1.5.0 // indirect github.com/buraksezer/consistent v0.10.0 // indirect + github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect @@ -245,7 +246,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260714120433-7667cad5ff5c // indirect @@ -360,7 +361,6 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/goterm v1.0.4 // indirect github.com/buger/jsonparser v1.2.0 // indirect - github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2 v2.70.2 // indirect @@ -634,7 +634,7 @@ require ( github.com/smartcontractkit/chainlink-aptos v0.0.0-20260708114855-e953eeb028a7 github.com/smartcontractkit/chainlink-automation v0.8.1 // indirect github.com/smartcontractkit/chainlink-ccip v0.1.1-solana.0.20260625091148-e5618f5682ee // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260715161014-611d8ac32364 github.com/smartcontractkit/chainlink-framework/capabilities v0.0.0-20260423135514-5b1a7565a99c // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260709082627-78ab5315e367 // indirect @@ -746,7 +746,7 @@ require ( google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.82.0 + google.golang.org/grpc v1.82.1 gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/guregu/null.v4 v4.0.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 52fbe06cc21..9cfab673f4e 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -337,8 +337,8 @@ github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= -github.com/bytecodealliance/wasmtime-go/v28 v28.0.0/go.mod h1:4OCU0xAW9ycwtX4nMF4zxwgJBJ5/0eMfJiHB0wAmkV4= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 h1:PAe4o7Mq1VjSx81Dc+V4w1XvxLouFTB384N4ZT4o+7k= +github.com/bytecodealliance/wasmtime-go/v47 v47.0.0/go.mod h1:icJLhTDV7EyKlZpeuBuJl4yoTMqhjUvNv1mewMJUvr4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -1756,12 +1756,12 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb/go.mod h1:67YbnoglYD61Pz/jTVCgav9wFq7S35OU8UyQSvPllRw= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100 h1:sH546tdm3VaYGaRLnZ6ZYBZiE25vEkMap76Xq/vQWx0= github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260715132147-59df1b138100/go.mod h1:0v6RGdYa9NezVnBPIAVyxB3A9furKWwaSkLha+URYGk= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba h1:1sx4ZrmoGiBOa0obx8K7VTojaODpXB67KGTfqLObjss= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260715163833-347adf48a9ba/go.mod h1:tAsXbMsNrwIp7vCqeauNsT4f3gVATbTmIviCkZPDgdo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d h1:xgOWExnIe7vZ9bOx57uliN5iJA3ooGDqftVNKui7NLQ= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260730141522-fb115ed8362d/go.mod h1:euYQ2WxVYGU2ivHXPBRpbU0Z3/SkekfIK5YPf7hdDY8= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6 h1:DvfhsiIxB4JMuR+r1UciKV8jKiwhI/OZI/QhKawC4pM= github.com/smartcontractkit/chainlink-common/keystore v1.2.1-0.20260623104656-f39eba3e2bc6/go.mod h1:6NefaCIMH4zFBN3T+cvsFGYoy3oTSPKDaxPAyBzlYTU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc h1:9xj2uDKZ4JSvJOfjT1LwoK1M9Ux+NUEE+FTMl4KqYsE= github.com/smartcontractkit/chainlink-data-streams v0.1.15-0.20260707105132-2da5d31f22fc/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -1802,8 +1802,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= @@ -2758,8 +2758,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 54689c52c7c77e944c9ad2b5bcd59f775fb44a5b Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 30 Jul 2026 16:48:31 -0400 Subject: [PATCH 12/14] fixing tests --- .../workflows/syncer/v2/handler_metering_test.go | 1 + docs/CONFIG.md | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index 8c052e8ba17..a6a8ad87129 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -224,6 +224,7 @@ func Test_meterRecords(t *testing.T) { WorkflowID: wfID.Hex(), Status: job.WorkflowSpecStatusActive, WorkflowOwner: "aabbccdd", + WorkflowName: "wf-name", }, }, newMeteringResourceManager(t, true, emitter)) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index ab7a3f7e6db..065170097be 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2674,13 +2674,7 @@ NodeID = '' # Default ``` Metering configures durable resource metering emission and the coarse deployment/node identity dimensions stamped on emitted MeterRecords and -MeterSnapshots. This TOML section is the single authority for metering on the -core node; there is no environment-variable gate. For capability LOOP plugins -the values are forwarded unchanged over the plugin environment -(loop.EnvConfig), which is only a child-process transport produced from this -config, not a separate gate. Snapshots are emitted on a fixed internal -interval and are bucket-aligned (each snapshot timestamp is truncated to the -interval) so cross-node snapshot buckets agree. +MeterSnapshots. ### MeterRecordsEnabled ```toml @@ -2730,7 +2724,6 @@ Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. NodeID = '' # Default ``` NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). -Required (non-empty) when `MeterRecordsEnabled = true`. ## CRE.Streams ```toml From d17cd8b31a6517fe0edf81d1f285db8683be7689 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Thu, 30 Jul 2026 21:28:45 -0400 Subject: [PATCH 13/14] cleaning up reconcile orphan behavior; adding paused state to workflow_specs_v2 to eliminate delta drift on register->pause->activate flow within billing dedup time bucket --- .../environment/chip_ingress_stack.go | 2 +- .../cre/confidential_relay_peerid_test.go | 1 - core/services/cre/cre.go | 1 - core/services/job/models.go | 1 + .../standard_capabilities_test.go | 2 +- core/services/workflows/artifacts/v2/orm.go | 63 ++-- .../workflows/artifacts/v2/orm_test.go | 14 +- core/services/workflows/artifacts/v2/store.go | 19 +- .../workflows/artifacts/v2/store_test.go | 2 +- core/services/workflows/syncer/v2/handler.go | 195 +++++++----- .../syncer/v2/handler_metering_test.go | 215 ++++++++++++-- .../workflows/syncer/v2/handler_test.go | 279 ++++++++++++++++-- core/services/workflows/syncer/v2/helpers.go | 4 + .../services/workflows/syncer/v2/mocks/orm.go | 73 ++++- .../workflows/syncer/v2/workflow_registry.go | 38 +-- .../syncer/v2/workflow_registry_test.go | 72 +++-- .../0302_workflow_specs_v2_registered_at.sql | 5 + 17 files changed, 771 insertions(+), 215 deletions(-) create mode 100644 core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql diff --git a/core/scripts/cre/environment/environment/chip_ingress_stack.go b/core/scripts/cre/environment/environment/chip_ingress_stack.go index 71260340c4e..f3d3bfca0fa 100644 --- a/core/scripts/cre/environment/environment/chip_ingress_stack.go +++ b/core/scripts/cre/environment/environment/chip_ingress_stack.go @@ -77,7 +77,7 @@ func schemaCommitRefFromGoMod(ctx context.Context, repoRoot, targetModule string // getSchemaSetFromGoMod resolves SchemaSets from chainlink-protos commits pinned in go.mod: // - workflows (chip-cre.json) for CRE/workflow telemetry // - node-platform (chip-schemas.json) for PluginRelayerConfigEmitter / common.v1.ChainPluginConfig -// - metering (chip-cll-meter.json) for durable resource metering (MeterRecord/MeterSnapshot on +// - metering (chip-cll.meter.json) for durable resource metering (MeterRecord/MeterSnapshot on // the cll.meter domain); without this, ChIP Ingress rejects those events at pre-publish encode // time with "Subject 'cll-meter-metering.v1.MeterRecord' not found" and drops them silently. func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, error) { diff --git a/core/services/cre/confidential_relay_peerid_test.go b/core/services/cre/confidential_relay_peerid_test.go index 47692762125..f81afc0393b 100644 --- a/core/services/cre/confidential_relay_peerid_test.go +++ b/core/services/cre/confidential_relay_peerid_test.go @@ -51,7 +51,6 @@ func (s stubConfig) Workflows() config.Workflows { return nil } func (s stubConfig) CRE() config.CRE { return nil } func (s stubConfig) P2P() config.P2P { return s.p2p } func (s stubConfig) Sharding() config.Sharding { return nil } -func (s stubConfig) Telemetry() config.Telemetry { return nil } func (s stubConfig) Metering() config.Metering { return nil } func peerIDFromByte(b byte) p2pkey.PeerID { diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index f8b5a5b8bd2..b2c3b9a299f 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -345,7 +345,6 @@ type Config interface { CRE() config.CRE P2P() config.P2P Sharding() config.Sharding - Telemetry() config.Telemetry Metering() config.Metering } diff --git a/core/services/job/models.go b/core/services/job/models.go index a669cd947ea..3dd92549e30 100644 --- a/core/services/job/models.go +++ b/core/services/job/models.go @@ -843,6 +843,7 @@ type WorkflowSpec struct { UpdatedAt time.Time `toml:"-" db:"updated_at"` SpecType WorkflowSpecType `toml:"spec_type" db:"spec_type"` Attributes []byte `db:"attributes"` + RegisteredAt int64 `toml:"-" db:"registered_at"` sdkWorkflow *sdk.WorkflowSpec rawSpec []byte config []byte diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index 72094b5b172..d9c24e98c9b 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -80,7 +80,7 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { require.Contains(t, err.Error(), capturingRegistrarErr) require.Empty(t, cfg.Env, - "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset; ") + "no operator-provided env vars should be forwarded when CL_CAPABILITIES_ENV is unset") }) t.Run("missing env file fails Start before RegisterLOOP", func(t *testing.T) { diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 5a618722209..8eb7a05b91b 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -3,6 +3,7 @@ package v2 import ( "context" "database/sql" + "errors" "time" "github.com/jmoiron/sqlx" @@ -25,8 +26,19 @@ type WorkflowSpecsDS interface { // workflow_owner); other fields are left zero. GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) - // DeleteWorkflowSpec deletes the workflow spec for the given workflow ID. - DeleteWorkflowSpec(ctx context.Context, id string) error + // DeleteWorkflowSpec deletes the spec row for id. Idempotent: a missing row + // returns (nil, nil), not an error. The deleted row is returned so the + // caller can derive the generation-scoped metering event_id from the row's + // own persisted registered_at (every node emits the identical id regardless + // of local staleness). + DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) + + // PauseWorkflowSpec tombstones the spec row: status becomes paused and the + // heavy artifact payload (hex binary + config) is cleared in the same + // statement, freeing storage while the row itself remains the durable record + // that this registration generation is still held (level-neutral for + // metering). Idempotent; pausing an absent or already-paused row is a no-op. + PauseWorkflowSpec(ctx context.Context, id string) error // DeleteWorkflowSpecs deletes workflow specs for the given workflow IDs in a single query. DeleteWorkflowSpecs(ctx context.Context, ids []string) error @@ -70,7 +82,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at, updated_at, spec_type, - attributes + attributes, + registered_at ) VALUES ( :workflow, :config, @@ -84,7 +97,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) :created_at, :updated_at, :spec_type, - :attributes + :attributes, + :registered_at ) ON CONFLICT (workflow_id) DO UPDATE SET workflow = EXCLUDED.workflow, @@ -98,7 +112,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, spec_type = EXCLUDED.spec_type, - attributes = EXCLUDED.attributes + attributes = EXCLUDED.attributes, + registered_at = EXCLUDED.registered_at RETURNING id ` @@ -135,11 +150,11 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp return &spec, nil } -// GetWorkflowSpecList returns all persisted workflow specs, projecting only the -// identity columns the metering snapshot path needs (workflow_id, workflow_owner). +// GetWorkflowSpecList returns all persisted workflow specs, projecting the +// identity columns needed by the orphan sweep and the metering snapshot path. func (orm *orm) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { query := ` - SELECT workflow_id, workflow_owner + SELECT workflow_id, workflow_owner, registered_at FROM workflow_specs_v2 ` @@ -151,27 +166,25 @@ func (orm *orm) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, e return specs, nil } -func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) error { - query := ` - DELETE FROM workflow_specs_v2 - WHERE workflow_id = $1 - ` - - result, err := orm.ds.ExecContext(ctx, query, id) - if err != nil { - return err +func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { + query := `DELETE FROM workflow_specs_v2 WHERE workflow_id = $1 RETURNING *` + var spec job.WorkflowSpec + err := orm.ds.GetContext(ctx, &spec, query, id) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil } - - rowsAffected, err := result.RowsAffected() if err != nil { - return err - } - - if rowsAffected == 0 { - return sql.ErrNoRows // No spec deleted + return nil, err } + return &spec, nil +} - return nil +func (orm *orm) PauseWorkflowSpec(ctx context.Context, id string) error { + query := `UPDATE workflow_specs_v2 + SET status = $2, workflow = '', config = '', updated_at = now() + WHERE workflow_id = $1` + _, err := orm.ds.ExecContext(ctx, query, id, job.WorkflowSpecStatusPaused) + return err } func (orm *orm) DeleteWorkflowSpecs(ctx context.Context, ids []string) error { diff --git a/core/services/workflows/artifacts/v2/orm_test.go b/core/services/workflows/artifacts/v2/orm_test.go index 63f78c3a1d1..677cdf1cfba 100644 --- a/core/services/workflows/artifacts/v2/orm_test.go +++ b/core/services/workflows/artifacts/v2/orm_test.go @@ -9,6 +9,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -194,8 +195,9 @@ func Test_DeleteWorkflowSpec(t *testing.T) { require.NoError(t, err) require.NotZero(t, id) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) + require.NotNil(t, deletedSpec) // Verify the record is deleted from the database var dbSpec job.WorkflowSpec @@ -204,16 +206,16 @@ func Test_DeleteWorkflowSpec(t *testing.T) { require.Equal(t, sql.ErrNoRows, err) }) - t.Run("fails if no workflow spec exists", func(t *testing.T) { + t.Run("returns false for non-existent workflow spec", func(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) ctx := t.Context() lggr := logger.TestLogger(t) orm := &orm{ds: db, lggr: lggr} - err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") - require.Error(t, err) - require.Equal(t, sql.ErrNoRows, err) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") + require.NoError(t, err) + assert.Nil(t, deletedSpec) }) } @@ -293,7 +295,7 @@ func Test_GetWorkflowSpec(t *testing.T) { require.NoError(t, err) require.Equal(t, spec.Workflow, dbSpec.Workflow) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + _, err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) }) diff --git a/core/services/workflows/artifacts/v2/store.go b/core/services/workflows/artifacts/v2/store.go index deb27c77632..77c6e641495 100644 --- a/core/services/workflows/artifacts/v2/store.go +++ b/core/services/workflows/artifacts/v2/store.go @@ -3,7 +3,6 @@ package v2 import ( "context" "crypto/sha256" - "database/sql" "encoding/base64" "encoding/hex" "errors" @@ -244,18 +243,14 @@ func (h *Store) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) return h.orm.UpsertWorkflowSpec(ctx, spec) } -// DeleteWorkflowArtifacts removes the workflow spec from the database. If not found, returns nil. -func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { - err := h.orm.DeleteWorkflowSpec(ctx, workflowID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - h.lggr.Warnw("failed to delete workflow spec: not found", "workflowID", workflowID) - return nil - } - return fmt.Errorf("failed to delete workflow spec: %w", err) - } +// DeleteWorkflowArtifacts removes the workflow spec from the database. If not +// found, returns (nil, nil). +func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { + return h.orm.DeleteWorkflowSpec(ctx, workflowID) +} - return nil +func (h *Store) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return h.orm.PauseWorkflowSpec(ctx, workflowID) } func (h *Store) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { diff --git a/core/services/workflows/artifacts/v2/store_test.go b/core/services/workflows/artifacts/v2/store_test.go index 59117a3a502..62ff154d520 100644 --- a/core/services/workflows/artifacts/v2/store_test.go +++ b/core/services/workflows/artifacts/v2/store_test.go @@ -87,7 +87,7 @@ func Test_Store_DeleteWorkflowArtifacts(t *testing.T) { require.NoError(t, err) // Delete the workflow artifacts by ID - err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) + _, err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) require.NoError(t, err) // Check that the workflow no longer exists diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 1e896c149c9..1ff2776787e 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -304,7 +304,8 @@ type WorkflowArtifactsStore interface { GetWorkflowSpec(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) - DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error + DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + PauseWorkflowArtifacts(ctx context.Context, workflowID string) error DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error } @@ -548,7 +549,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { var err error defer func() { - if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, err); err2 != nil { + if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, customerFacingError(err)); err2 != nil { h.lggr.Errorf("failed to emit status changed event: %+v", err2) } }() @@ -576,19 +577,17 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { // Get workflow spec from database to get owner and name info for organization lookup, // and to check if the spec exists to determine if a MeterRecord is warranted. var wfOwner, wfName, orgID string - specExisted := false if spec, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID); gerr == nil && spec != nil { - specExisted = true wfOwner = spec.WorkflowOwner wfName = spec.WorkflowName } else if gerr != nil && !errors.Is(gerr, sql.ErrNoRows) { - h.lggr.Warnw("failed to read workflow spec during deletion, proceeding without metadata", "workflowID", wfID, "error", gerr) + h.lggr.Errorw("failed to read workflow spec during deletion, proceeding without metadata", "workflowID", wfID, "error", gerr) } if wfOwner != "" { - if oerr, ferr := h.fetchOrganizationID(ctx, wfOwner); ferr != nil { - h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", ferr) + if resolvedOrgID, orgErr := h.fetchOrganizationID(ctx, wfOwner); orgErr != nil { + h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", orgErr) } else { - orgID = oerr + orgID = resolvedOrgID } } ctx = contexts.WithCRE(ctx, contexts.CRE{Org: orgID, Owner: wfOwner, Workflow: wfID}) @@ -612,7 +611,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { } }() - if herr = h.workflowDeletedEvent(ctx, payload, WorkflowDeleted, wfOwner, specExisted); herr != nil { + if herr = h.workflowDeletedEvent(ctx, payload, wfOwner); herr != nil { if errors.Is(herr, ErrDrainInProgress) { logCustMsg(ctx, cma, fmt.Sprintf("workflow deletion deferred: %v", herr), h.lggr) } else { @@ -669,6 +668,7 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } + h.emitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) @@ -689,13 +689,32 @@ func (h *eventHandler) workflowRegisteredEvent( spec = newSpec case spec.Status != status: - spec.Status = status - if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { - return fmt.Errorf("failed to update workflow spec: %w", innerErr) + if status == job.WorkflowSpecStatusActive && spec.Workflow == "" { + // Activating a paused tombstone: no delta. + newSpec, innerErr := h.createWorkflowSpec(ctx, payload) + if innerErr != nil { + return innerErr + } + spec = newSpec + } else { + spec.Status = status + if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { + spec.RegisteredAt = int64(payload.CreatedAt) + } + if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { + return fmt.Errorf("failed to update workflow spec: %w", innerErr) + } + // Status-only flip: no artifact-persistence transition, no delta. + } + } + + // Legacy-row convergence: backfill registered_at for pre-migration rows + // (at most once per row). Fail-open: a write error is logged, not returned. + if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { + spec.RegisteredAt = int64(payload.CreatedAt) + if _, err := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); err != nil { + h.lggr.Warnw("failed to backfill registered_at", "workflowID", spec.WorkflowID, "err", err) } - // Status-only update (e.g. activate an already-stored spec): the spec - // stays stored, so there is no artifact-persistence transition and no - // meter delta. The level is unchanged and remains covered by snapshots. } // Next, let's synchronize the engine. @@ -786,6 +805,7 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, Attributes: payload.Attributes, + RegisteredAt: int64(payload.CreatedAt), } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -940,46 +960,17 @@ func (h *eventHandler) createEngineModule( return engineModule } -// workflowPausedEvent handles the WorkflowPausedEvent event type. This method must remain idempotent. -// -// Pause emits no metering delta (by design): it removes artifacts and stops the -// engine, but the durable spec's release is realized by its absence from -// subsequent snapshots, not a -1 delta. Activate-after-pause re-runs the create -// path and emits a +1 with the register event_id (same on-chain CreatedAt), so a -// within-bucket pause→activate merges with the original register; cumulative-delta -// drift is corrected by snapshot level reconciliation. -func (h *eventHandler) workflowPausedEvent( - ctx context.Context, - payload WorkflowPausedEvent, -) error { - return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}, WorkflowPaused, "", false) -} - -// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. -// originatingEvent names the event that triggered this call (e.g. WorkflowPaused -// when delegated from workflowPausedEvent). deleteOwner/specExisted are supplied -// by the caller (Handle pre-reads the spec once for labels and to decide whether -// a -1 is warranted); pause passes ""/false since it emits no delta. -func (h *eventHandler) workflowDeletedEvent( - ctx context.Context, - payload WorkflowDeletedEvent, - originatingEvent WorkflowRegistryEventName, - deleteOwner string, - specExisted bool, -) error { - // The order in the handler is slightly different to the order in `tryEngineCleanup`. - // This is because the engine requires its corresponding DB record to be present to be successfully - // closed. - // At the same time, popping the engine should occur last to allow deletes to be retried if any of the - // prior steps fail. - workflowID := payload.WorkflowID.Hex() - e, ok := h.engineRegistry.Get(payload.WorkflowID) +// stopEngine drains (returning ErrDrainInProgress while executions remain) and +// closes the engine for workflowID if one is registered. +// Returns the drainable handle (nil-able) for drain-completed metrics. +func (h *eventHandler) stopEngine(ctx context.Context, workflowID types.WorkflowID) (DrainableService, error) { + e, ok := h.engineRegistry.Get(workflowID) var drainable DrainableService - var isDrainable bool if ok { + var isDrainable bool if drainable, isDrainable = e.Service.(DrainableService); isDrainable { if started := drainable.Drain(); started { - h.lggr.Infow("initiated drain for workflow engine", "workflowID", workflowID) + h.lggr.Infow("initiated drain for workflow in workflow engine", "workflowID", workflowID.String()) if h.metrics != nil { h.metrics.incrementDrainStarted(ctx) } @@ -990,42 +981,92 @@ func (h *eventHandler) workflowDeletedEvent( h.metrics.incrementDeleteDeferred(ctx, "drain_in_progress") } h.lggr.Infow("workflow deletion deferred: active executions still running", - "workflowID", workflowID, + "workflowID", workflowID.String(), "activeExecutions", active) - return fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) + return nil, fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) } } if innerErr := e.Close(); innerErr != nil && !errors.Is(innerErr, services.ErrAlreadyStopped) { - return fmt.Errorf("failed to close workflow engine: %w", innerErr) + return nil, fmt.Errorf("failed to close workflow engine: %w", innerErr) } } + return drainable, nil +} - if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { +// releaseSpecStorage deletes the persisted spec row and, iff a row was +// actually removed, emits the -1 generation delta. RowsAffected is the +// exactly-once gate: redeliveries observe no row and emit nothing; a transient +// DELETE error returns before emission so the retry emits when the delete +// lands. The event_id is derived AFTER the delete from the row's persisted +// registered_at, so all nodes emit the identical id. Module-cache cleanup +// always runs. owner may be empty; org resolution is fail-open. +func (h *eventHandler) releaseSpecStorage(ctx context.Context, workflowID, owner string) error { + deletedSpec, err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, workflowID) + if err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } - // A real delete that removed a previously-persisted spec emits a -1 delta. - // Gating on specExisted prevents a redelivered delete (spec already removed, - // and DeleteWorkflowArtifacts maps ErrNoRows→nil) from emitting a second -1. - // Pause is delegated here too (originatingEvent == WorkflowPaused); it passes - // specExisted=false to avoid emitting a delta - if originatingEvent == WorkflowDeleted && specExisted { - // eventID must be consistent across DONs - h.emitSpecDelta(ctx, -1, workflowID, deleteOwner, - resourcemanager.EventID("workflow-spec-delete", workflowID)) + if deletedSpec != nil { + parts := []string{workflowID} + if deletedSpec.RegisteredAt > 0 { + parts = append(parts, strconv.FormatInt(deletedSpec.RegisteredAt, 10)) + } + h.emitSpecDelta(ctx, -1, workflowID, owner, resourcemanager.EventID("workflow-spec-delete", parts...)) } + h.cleanupModuleCache(workflowID) + return nil +} - h.cleanupModuleCache(payload.WorkflowID.Hex()) +// workflowPausedEvent handles WorkflowPaused. Idempotent. +// +// Pause is level-neutral for metering: the spec row survives as a tombstone +// (status=paused, artifact payload cleared to free storage) so the +// registration generation stays metered until deletion — mirroring the +// on-chain registry, which retains the paused registration. Emitting per-cycle +// pause/activate deltas is impossible without drift: no DON-consistent +// per-occurrence discriminator exists, so a -1 here would force the +// re-activation +1 to reuse the register event_id and be dropped by consumer +// dedup. Deltas therefore fire only at generation boundaries (register/delete). +func (h *eventHandler) workflowPausedEvent( + ctx context.Context, + payload WorkflowPausedEvent, +) error { + workflowID := payload.WorkflowID.Hex() + if _, err := h.stopEngine(ctx, payload.WorkflowID); err != nil { + return err + } + if err := h.workflowArtifactsStore.PauseWorkflowArtifacts(ctx, workflowID); err != nil { + return fmt.Errorf("failed to pause workflow artifacts: %w", err) + } + h.cleanupModuleCache(workflowID) + if _, err := h.engineRegistry.Pop(payload.WorkflowID); err != nil && !errors.Is(err, ErrNotFound) { + return err + } + return nil +} - _, err := h.engineRegistry.Pop(payload.WorkflowID) +// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. +func (h *eventHandler) workflowDeletedEvent( + ctx context.Context, + payload WorkflowDeletedEvent, + owner string, +) error { + workflowID := payload.WorkflowID.Hex() + drainable, err := h.stopEngine(ctx, payload.WorkflowID) + if err != nil { + return err + } + if err := h.releaseSpecStorage(ctx, workflowID, owner); err != nil { + return err + } + _, err = h.engineRegistry.Pop(payload.WorkflowID) if errors.Is(err, ErrNotFound) { return nil } if err != nil { return err } - - if isDrainable { + if drainable != nil { startedAt, exists := drainable.DrainStartedAt() if exists && h.metrics != nil { h.metrics.recordDrainCompleted(ctx, time.Since(startedAt)) @@ -1034,6 +1075,13 @@ func (h *eventHandler) workflowDeletedEvent( return nil } +// ReleaseOrphanedSpec implements the workflow registry's downtime-orphan sweep +// contract: release the persisted spec for a workflow absent from all sources' +// metadata with no running engine. +func (h *eventHandler) ReleaseOrphanedSpec(ctx context.Context, workflowID, owner string) error { + return h.releaseSpecStorage(ctx, workflowID, owner) +} + // emitSpecDelta emits one metering.v1.MeterRecord (METER_ACTION_UPDATE) // capturing a signed ±delta to the durable workflow_specs_v2 level for // workflowID. Each record is a first-class billing event: a +1 when a new @@ -1098,22 +1146,17 @@ func (h *eventHandler) ResourceIdentity() resourcemanager.ResourceIdentity { return h.baseIdentity() } -// GetWorkflowSpecList allows meter deltas to ignore activate-then-pause and -// pause-then-activate events. +// GetWorkflowSpecList backs the orphan sweep and the metering snapshot path. func (h *eventHandler) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { return h.workflowArtifactsStore.GetWorkflowSpecList(ctx) } // GetUtilization implements resourcemanager.Meterable: it returns one -// SnapshotEntry per persisted workflow_specs_v2 spec. +// SnapshotEntry per persisted workflow_specs_v2 spec. // The durable resource is the stored spec, NOT the running engine. -// Why: paused-but-stored specs and -// specs whose engine failed to start are still accounted for, and a delete is -// reflected by the spec's absence from the next snapshot. // // resource_id is the workflow_id; the organization is resolved fail-open from -// the spec's stored owner through the org resolver. Fail-open: a store error -// yields no entries for this tick. +// the spec's stored owner through the org resolver. Must fail open. func (h *eventHandler) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { specs, err := h.workflowArtifactsStore.GetWorkflowSpecList(ctx) if err != nil { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index a6a8ad87129..f58f460c439 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -21,6 +21,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" "github.com/smartcontractkit/chainlink/v2/core/capabilities" @@ -34,6 +35,17 @@ import ( v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" ) +// meteringwfID is the workflow ID that GenerateWorkflowID produces from the +// stub store's FetchWorkflowArtifacts return values (binary="binary", +// config="config") with owner 0xaabbccdd and name "wf-name". Tests that use +// persistUpserts and register as Active must use this ID so tryEngineCreate's +// validation passes. +var meteringwfID = func() types.WorkflowID { + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + wfIDBytes, _ := pkgworkflows.GenerateWorkflowID(owner, "wf-name", []byte("binary"), []byte("config"), "") + return types.WorkflowID(wfIDBytes) +}() + // recordingEmitter is a fake resourcemanager.Emitter that decodes and stores // every emitted MeterRecord. If err is set, Emit fails instead. type recordingEmitter struct { @@ -272,7 +284,7 @@ func Test_meterRecords(t *testing.T) { }, }, newMeteringResourceManager(t, true, emitter)) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) records := emitter.Records() require.Len(t, records, 1) @@ -308,7 +320,7 @@ func Test_meterRecords(t *testing.T) { deleteErr: assert.AnError, }, newMeteringResourceManager(t, true, emitter)) - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") require.ErrorIs(t, err, assert.AnError) assert.Empty(t, emitter.Records()) }) @@ -329,12 +341,12 @@ func Test_meterRecords(t *testing.T) { drainable.activeExecutions.Store(1) require.NoError(t, h.engineRegistry.Add(wfID, "test-source", drainable)) - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") require.ErrorIs(t, err, ErrDrainInProgress) assert.Empty(t, emitter.Records()) drainable.activeExecutions.Store(0) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) records := emitter.Records() require.Len(t, records, 1) @@ -360,7 +372,7 @@ func Test_meterRecords(t *testing.T) { WorkflowOwner: wfOwner, WorkflowName: "wf-name", })) - require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, WorkflowDeleted, "aabbccdd", true)) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) assert.Empty(t, emitter.Records()) }) @@ -521,22 +533,18 @@ func Test_meterRecords_RedeliveredDeleteEmitsExactlyOneDelta(t *testing.T) { resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) } -// activate-after-pause re-runs the create path and emits a +1 whose -// event_id equals the original register event_id (same on-chain CreatedAt). -// Pause itself emits no delta; within-bucket pause→activate therefore merges -// with the original register, with cumulative-delta drift corrected by snapshots. -func Test_meterRecords_ActivateAfterPauseEmitsRegisterEventID(t *testing.T) { +// Pause and activate are level-neutral: neither emits a metering delta. +// The +1 from register stays as the sole record through a pause→activate cycle. +func Test_meterRecords_PauseActivateCycleIsLevelNeutral(t *testing.T) { t.Parallel() emitter := &recordingEmitter{} - // The stub does not persist on upsert, so register and activate each take - // the new-spec path; pause deletes (a no-op here) and emits nothing. - h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) - wfID := types.WorkflowID{42} + wfID := meteringwfID createdAt := uint64(123) wantEventID := resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10)) payload := WorkflowRegisteredEvent{ - Status: WorkflowStatusPaused, + Status: WorkflowStatusActive, WorkflowID: wfID, WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, WorkflowName: "wf-name", @@ -544,15 +552,14 @@ func Test_meterRecords_ActivateAfterPauseEmitsRegisterEventID(t *testing.T) { } require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.Len(t, emitter.Records(), 1) + requireSpecDelta(t, emitter.Records()[0], "1", wfID.Hex(), wantEventID) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) - require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.Len(t, emitter.Records(), 1, "pause must not emit a delta") - records := emitter.Records() - require.Len(t, records, 2, "register and activate each emit one +1; pause emits none") - requireSpecDelta(t, records[0], "1", wfID.Hex(), wantEventID) - requireSpecDelta(t, records[1], "1", wfID.Hex(), wantEventID) - assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId(), - "activate-after-pause must reuse the register event_id (same on-chain CreatedAt)") + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.Len(t, emitter.Records(), 1, "activate-after-pause must not emit a delta") } // OrgId is resolved from the workflow owner and stamped on emitted records. @@ -629,3 +636,167 @@ func Test_meterRecords_TransientDBErrorOnGetSpecEmitsNoDelta(t *testing.T) { require.Error(t, err) assert.Empty(t, emitter.Records(), "transient DB error must not take the new-spec path or emit a +1") } + +// A full register→pause→activate→delete cycle emits exactly two records: +// one +1 at register and one -1 at delete. Pause and activate emit nothing. +// The delete event_id carries the registered_at that flowed insert→RETURNING, +// proving the generation-scoped id is DON-consistent. +func Test_meterRecords_FullLifecycleEmitsExactlyTwoRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(123) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly two records: +1 at register, -1 at delete") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} + +// A transient delete error returns before emission (no -1); a successful retry +// emits exactly one -1. This fixes the lost-−1 hazard from the old pre-read gate. +func Test_meterRecords_TransientDeleteErrorThenRetryEmitsOnce(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + deleteErr: assert.AnError, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{77} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + require.Len(t, emitter.Records(), 1, "register emits +1") + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()[1:], "failed delete must not emit a -1") + + store.deleteErr = nil + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly one -1 after successful retry") + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), "1")) +} + +// A legacy row with RegisteredAt == 0 produces a delete event_id with no +// timestamp part (the fallback for pre-migration rows). +func Test_meterRecords_LegacyRowDeleteUsesFallbackID(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{88}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + RegisteredAt: 0, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: types.WorkflowID{88}}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", types.WorkflowID{88}.Hex(), + resourcemanager.EventID("workflow-spec-delete", types.WorkflowID{88}.Hex())) +} + +// Identical event sequences with RM nil / disabled / erroring emitter produce +// identical row+engine outcomes and zero handler errors (fail-open equivalence). +func Test_meterRecords_FailOpenEquivalence(t *testing.T) { + t.Parallel() + wfID := meteringwfID + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: 1, + } + delEvt := WorkflowDeletedEvent{WorkflowID: wfID} + + runCycle := func(t *testing.T, rm *resourcemanager.ResourceManager, emitter resourcemanager.Emitter) (*stubWorkflowArtifactsStore, *eventHandler) { + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, rm) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), delEvt, "aabbccdd")) + return store, h + } + + // RM nil + nilStore, nilH := runCycle(t, nil, nil) + assert.Nil(t, nilH.resourceManager) + assert.Nil(t, nilStore.spec, "spec should be deleted by the cycle") + + // RM disabled + disabledEmitter := &recordingEmitter{} + disabledRM := newMeteringResourceManager(t, false, disabledEmitter) + disabledStore, disabledH := runCycle(t, disabledRM, disabledEmitter) + assert.NotNil(t, disabledH.resourceManager) + assert.Empty(t, disabledEmitter.Records(), "disabled RM emits nothing") + assert.Nil(t, disabledStore.spec, "spec should be deleted by the cycle") + + // RM enabled but erroring emitter + errEmitter := &recordingEmitter{err: errors.New("beholder unavailable")} + errRM := newMeteringResourceManager(t, true, errEmitter) + errStore, errH := runCycle(t, errRM, errEmitter) + assert.NotNil(t, errH.resourceManager) + assert.Empty(t, errEmitter.Records(), "erroring emitter stores nothing") + assert.Nil(t, errStore.spec, "spec should be deleted by the cycle") +} + +// The orphan sweep releases a paused tombstone (no engine) with exactly one -1 +// carrying the generation delete-id. This is the sweep's new load-bearing case: +// workflows deleted on-chain while paused have a tombstone but no engine. +func Test_meterRecords_SweepReleasesPausedTombstone(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(456) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + })) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + // After pause the engine is popped; the tombstone has no engine → sweep path. + require.NoError(t, h.ReleaseOrphanedSpec(t.Context(), wfID.Hex(), "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "register +1 and sweep -1") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index f1bb65a75f5..fcf1f36c551 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -9,6 +9,7 @@ import ( "fmt" "math/big" "net" + "strconv" "sync/atomic" "testing" "time" @@ -39,6 +40,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder/beholdertest" "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -1021,13 +1023,17 @@ func (m *mockArtifactStore) UpsertWorkflowSpec(ctx context.Context, spec *job.Wo return m.artifactStore.UpsertWorkflowSpec(ctx, spec) } -func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { +func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { if m.deleteWorkflowArtifactsErr != nil { - return m.deleteWorkflowArtifactsErr + return nil, m.deleteWorkflowArtifactsErr } return m.artifactStore.DeleteWorkflowArtifacts(ctx, workflowID) } +func (m *mockArtifactStore) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return m.artifactStore.PauseWorkflowArtifacts(ctx, workflowID) +} + func (m *mockArtifactStore) GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) { return m.artifactStore.GetWorkflowSpecList(ctx) } @@ -1136,7 +1142,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1191,7 +1197,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1292,7 +1298,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent, WorkflowDeleted, "", false) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.Error(t, err, failWith) // Verify the record is still in the DB @@ -1306,17 +1312,21 @@ func Test_workflowDeletedHandler(t *testing.T) { } type stubWorkflowArtifactsStore struct { - spec *job.WorkflowSpec - specs []*job.WorkflowSpec - upsertErr error - deleteErr error - listErr error - getSpecErr error - deleteCalls atomic.Int32 + spec *job.WorkflowSpec + specs []*job.WorkflowSpec + persistUpserts bool + upsertErr error + deleteErr error + listErr error + getSpecErr error + deleteCalls atomic.Int32 + pauseCalls atomic.Int32 + fetchCalls atomic.Int32 } func (s *stubWorkflowArtifactsStore) FetchWorkflowArtifacts(context.Context, string, string, string) ([]byte, []byte, error) { - return nil, nil, nil + s.fetchCalls.Add(1) + return []byte("binary"), []byte("config"), nil } func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (*job.WorkflowSpec, error) { @@ -1329,10 +1339,14 @@ func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (* return s.spec, nil } -func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(context.Context, *job.WorkflowSpec) (int64, error) { +func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(_ context.Context, spec *job.WorkflowSpec) (int64, error) { if s.upsertErr != nil { return 0, s.upsertErr } + if s.persistUpserts { + cp := *spec + s.spec = &cp + } return 1, nil } @@ -1343,14 +1357,26 @@ func (s *stubWorkflowArtifactsStore) GetWorkflowSpecList(context.Context) ([]*jo return s.specs, nil } -func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) error { +func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) (*job.WorkflowSpec, error) { s.deleteCalls.Add(1) if s.deleteErr != nil { - return s.deleteErr + return nil, s.deleteErr + } + if s.spec == nil { + return nil, nil } - // Mimic the real store: a successful delete removes the spec, so a - // redelivered delete observes no row (sql.ErrNoRows) on the next read. + deleted := *s.spec s.spec = nil + return &deleted, nil +} + +func (s *stubWorkflowArtifactsStore) PauseWorkflowArtifacts(context.Context, string) error { + s.pauseCalls.Add(1) + if s.spec != nil { + s.spec.Status = job.WorkflowSpecStatusPaused + s.spec.Workflow = "" + s.spec.Config = "" + } return nil } @@ -1374,7 +1400,7 @@ func Test_workflowDeletedEvent_DrainInProgress(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted, "", false) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.Error(t, err) require.ErrorIs(t, err, ErrDrainInProgress) assert.Equal(t, int32(1), drainable.drainCalls.Load()) @@ -1400,7 +1426,7 @@ func Test_workflowDeletedEvent_IgnoresErrAlreadyStopped(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, WorkflowDeleted, "", false) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.NoError(t, err) assert.Equal(t, int32(1), drainable.closeCalls.Load()) assert.Equal(t, int32(1), artifactStore.deleteCalls.Load()) @@ -1740,7 +1766,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold } mockDeleteORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(spec, nil) - mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil) + mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(&job.WorkflowSpec{}, nil) deleteArtifactStore, err := artifacts.NewStore(lggr, mockDeleteORM, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ ArtifactStorageHost: "example.com", @@ -1908,3 +1934,214 @@ func (c *confidentialCap) Initialise(_ context.Context, _ core.StandardCapabilit } var _ server.ClientCapability = &confidentialCap{} + +// assertStubState checks the in-memory stub's spec state. +func assertStubState(t *testing.T, s *stubWorkflowArtifactsStore, exists bool, status job.WorkflowSpecStatus, artifactsEmpty bool, registeredAt int64) { + t.Helper() + if !exists { + assert.Nil(t, s.spec, "spec should not exist") + return + } + require.NotNil(t, s.spec, "spec should exist") + assert.Equal(t, status, s.spec.Status) + if artifactsEmpty { + assert.Empty(t, s.spec.Workflow, "workflow artifacts should be empty") + assert.Empty(t, s.spec.Config, "config artifacts should be empty") + } else { + assert.NotEmpty(t, s.spec.Workflow, "workflow artifacts should be present") + } + assert.Equal(t, registeredAt, s.spec.RegisteredAt) +} + +// Test_specStorage_StateMachine exercises the storage state machine across +// (row state × event) transitions, verifying row existence, Status, artifact +// presence, RegisteredAt, engine-registry contents, and fetch call counts. +func Test_specStorage_StateMachine(t *testing.T) { + t.Parallel() + + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + hexWorkflow := hex.EncodeToString(binaryData) + createdAt := uint64(999) + + makeHandler := func(store *stubWorkflowArtifactsStore) *eventHandler { + return newMeteringTestHandler(t, store, nil) + } + activePayload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + t.Run("absent + Activated → fetch, insert, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, int32(1), store.fetchCalls.Load(), "exactly one fetch") + }) + + t.Run("active + Paused → tombstone, engine popped; redelivery no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be popped") + // Redelivery + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + }) + + t.Run("tombstone + Activated → fetch, full restore, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + fetchBefore := store.fetchCalls.Load() + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, fetchBefore+1, store.fetchCalls.Load(), "exactly one new fetch for restore") + }) + + t.Run("tombstone + Deleted → row removed, engine steps no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be absent") + }) + + t.Run("paused+artifacts + Activated → status flip, no fetch", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusPaused, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: int64(createdAt), + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + }) + + t.Run("absent + Paused → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("absent + Deleted → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "")) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("active RegisteredAt==0 + Activated (no status change) → backfill", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: 0, + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + }) +} + +// Test_handler_SourceParity_PauseActivateCycle runs a full register→pause→ +// activate→delete cycle twice with contract-style and gRPC-style source +// identifiers, asserting identical storage outcomes, engine-registry source +// scoping, and identical metering records/ids. +func Test_handler_SourceParity_PauseActivateCycle(t *testing.T) { + t.Parallel() + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + createdAt := uint64(777) + sources := []string{ + "contract:42:0xabcdef1234567890", + "grpc:centralized-registry:v1", + } + + for _, source := range sources { + t.Run(source, func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + Source: source, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + // Verify engine is registered with the correct source + entry, ok := h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID, Source: source})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + entry, ok = h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID, Source: source}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + + // Both sources produce identical metering: one +1, one -1, same ids + records := emitter.Records() + require.Len(t, records, 2) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + }) + } +} diff --git a/core/services/workflows/syncer/v2/helpers.go b/core/services/workflows/syncer/v2/helpers.go index 2ebe70f0cb5..cefb2b220fb 100644 --- a/core/services/workflows/syncer/v2/helpers.go +++ b/core/services/workflows/syncer/v2/helpers.go @@ -83,6 +83,10 @@ func (m *testEvtHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSp return nil, nil } +func (m *testEvtHandler) ReleaseOrphanedSpec(context.Context, string, string) error { + return nil +} + func (m *testEvtHandler) ClearEvents() { m.mux.Lock() defer m.mux.Unlock() diff --git a/core/services/workflows/syncer/v2/mocks/orm.go b/core/services/workflows/syncer/v2/mocks/orm.go index 67c36249cd5..5a6e40bdbfb 100644 --- a/core/services/workflows/syncer/v2/mocks/orm.go +++ b/core/services/workflows/syncer/v2/mocks/orm.go @@ -23,21 +23,33 @@ func (_m *ORM) EXPECT() *ORM_Expecter { } // DeleteWorkflowSpec provides a mock function with given fields: ctx, id -func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) error { +func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteWorkflowSpec") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + var r0 *job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*job.WorkflowSpec, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *job.WorkflowSpec); ok { r0 = rf(ctx, id) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*job.WorkflowSpec) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // ORM_DeleteWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteWorkflowSpec' @@ -59,12 +71,59 @@ func (_c *ORM_DeleteWorkflowSpec_Call) Run(run func(ctx context.Context, id stri return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 error) *ORM_DeleteWorkflowSpec_Call { +func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 *job.WorkflowSpec, _a1 error) *ORM_DeleteWorkflowSpec_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) (*job.WorkflowSpec, error)) *ORM_DeleteWorkflowSpec_Call { + _c.Call.Return(run) + return _c +} + +// PauseWorkflowSpec provides a mock function with given fields: ctx, id +func (_m *ORM) PauseWorkflowSpec(ctx context.Context, id string) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for PauseWorkflowSpec") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ORM_PauseWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PauseWorkflowSpec' +type ORM_PauseWorkflowSpec_Call struct { + *mock.Call +} + +// PauseWorkflowSpec is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *ORM_Expecter) PauseWorkflowSpec(ctx interface{}, id interface{}) *ORM_PauseWorkflowSpec_Call { + return &ORM_PauseWorkflowSpec_Call{Call: _e.mock.On("PauseWorkflowSpec", ctx, id)} +} + +func (_c *ORM_PauseWorkflowSpec_Call) Run(run func(ctx context.Context, id string)) *ORM_PauseWorkflowSpec_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) Return(_a0 error) *ORM_PauseWorkflowSpec_Call { _c.Call.Return(_a0) return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_DeleteWorkflowSpec_Call { +func (_c *ORM_PauseWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_PauseWorkflowSpec_Call { _c.Call.Return(run) return _c } diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index 88abf98bfe5..ffb5af4c8cc 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -145,6 +145,7 @@ type evtHandler interface { Handle(ctx context.Context, event Event) error EmitActivationAbandoned(ctx context.Context, event Event, reason eventsv2.ActivationAbandonReason, activationErr error, retryCount int32) error GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) + ReleaseOrphanedSpec(ctx context.Context, workflowID, owner string) error } type donNotifier interface { @@ -1015,13 +1016,6 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } - // Downtime-orphan sweep: after every source has been reconciled (and - // only when all fetched successfully, so the metadata union is - // complete), delete persisted specs whose workflowID is absent from - // all sources. This catches workflows deleted on-chain while this - // node was down — their spec rows linger with no engine and no - // source event. Dispatched through Handle, they take the existing - // delete path (artifact cleanup + -1 delta) with no new emission code. if !sourceFetchFailed && len(w.workflowSources) > 0 { w.reconcileOrphanedSpecs(ctx, allMetadataIDs) } @@ -1047,15 +1041,11 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) } } -// reconcileOrphanedSpecs dispatches a WorkflowDeleted event for every persisted -// workflow spec whose workflowID is absent from liveWorkflowIDs (the union of -// every source's metadata this tick). It uses EXISTING state — the persisted -// spec rows are this node's record of "workflows it believes exist" — and the -// registry metadata is the source of truth. Each orphan flows through the -// existing handler delete path, producing the -1 delta and artifact cleanup with -// zero new emission code. Paused workflows remain in metadata (status Paused), -// so they are present in liveWorkflowIDs and are never swept. Fail-open: a -// listing error or an unparseable id skips the sweep / that row. +// reconcileOrphanedSpecs releases persisted specs whose workflowID is +// absent from all sources. This catches workflows deleted while this +// node is down or restarting. Released through ReleaseOrphanedSpec, they take the +// existing delete path (artifact cleanup and -1 metering delta) but must +// not emit workflow status changed events. func (w *workflowRegistry) reconcileOrphanedSpecs(ctx context.Context, liveWorkflowIDs map[string]struct{}) { specs, err := w.handler.GetWorkflowSpecList(ctx) if err != nil { @@ -1076,15 +1066,15 @@ func (w *workflowRegistry) reconcileOrphanedSpecs(ctx context.Context, liveWorkf } var wfID wftypes.WorkflowID copy(wfID[:], wfIDBytes) - w.lggr.Debugw("orphaned workflow spec: persisted locally but absent from registry metadata; dispatching delete", - "workflowID", spec.WorkflowID, "owner", spec.WorkflowOwner) - evt := Event{ - Name: WorkflowDeleted, - Data: WorkflowDeletedEvent{WorkflowID: wfID}, - Head: Head{}, + if _, engineFound := w.engineRegistry.Get(wfID); engineFound { + // Engine-owned: the reconciliation path deletes it with drain/retry + // machinery. The sweep only handles engine-less leftovers. + continue } - if herr := w.handler.Handle(ctx, evt); herr != nil { - w.lggr.Warnw("failed to handle orphaned workflow spec delete", "workflowID", spec.WorkflowID, "err", herr) + w.lggr.Debugw("orphaned workflow spec: persisted locally but absent from registry metadata; releasing", + "workflowID", spec.WorkflowID, "owner", spec.WorkflowOwner) + if rerr := w.handler.ReleaseOrphanedSpec(ctx, spec.WorkflowID, spec.WorkflowOwner); rerr != nil { + w.lggr.Warnw("failed to release orphaned workflow spec", "workflowID", spec.WorkflowID, "err", rerr) } } } diff --git a/core/services/workflows/syncer/v2/workflow_registry_test.go b/core/services/workflows/syncer/v2/workflow_registry_test.go index b178999d6fa..9317b3c1a57 100644 --- a/core/services/workflows/syncer/v2/workflow_registry_test.go +++ b/core/services/workflows/syncer/v2/workflow_registry_test.go @@ -2001,13 +2001,19 @@ func TestWorkflowRegistry_getTicker_WithTickerOverride(t *testing.T) { require.Equal(t, (<-chan time.Time)(customCh), tickerCh) } +type orphanRelease struct { + workflowID string + owner string +} + // orphanSweepFakeHandler is a minimal evtHandler that returns a fixed spec list -// and records every Handle call, so reconcileOrphanedSpecs can be unit-tested -// in isolation. +// and records every Handle / ReleaseOrphanedSpec call, so reconcileOrphanedSpecs +// can be unit-tested in isolation. type orphanSweepFakeHandler struct { - mu sync.Mutex - specs []*job.WorkflowSpec - handled []Event + mu sync.Mutex + specs []*job.WorkflowSpec + handled []Event + released []orphanRelease } func (h *orphanSweepFakeHandler) Close() error { return nil } @@ -2024,6 +2030,12 @@ func (h *orphanSweepFakeHandler) EmitActivationAbandoned(context.Context, Event, func (h *orphanSweepFakeHandler) GetWorkflowSpecList(context.Context) ([]*job.WorkflowSpec, error) { return h.specs, nil } +func (h *orphanSweepFakeHandler) ReleaseOrphanedSpec(_ context.Context, workflowID, owner string) error { + h.mu.Lock() + defer h.mu.Unlock() + h.released = append(h.released, orphanRelease{workflowID: workflowID, owner: owner}) + return nil +} func (h *orphanSweepFakeHandler) Handled() []Event { h.mu.Lock() defer h.mu.Unlock() @@ -2031,12 +2043,19 @@ func (h *orphanSweepFakeHandler) Handled() []Event { copy(cp, h.handled) return cp } +func (h *orphanSweepFakeHandler) Released() []orphanRelease { + h.mu.Lock() + defer h.mu.Unlock() + cp := make([]orphanRelease, len(h.released)) + copy(cp, h.released) + return cp +} // a persisted spec whose workflowID is absent from the registry metadata -// (the union of all sources) is a downtime orphan — it is dispatched through -// the existing delete path. Specs still present in metadata (including paused -// workflows, which remain in metadata) are left alone. -func Test_reconcileOrphanedSpecs_DispatchesDeleteForOrphansOnly(t *testing.T) { +// (the union of all sources) and has no running engine is a downtime orphan — +// it is released through ReleaseOrphanedSpec. Specs still present in metadata +// (including paused workflows, which remain in metadata) are left alone. +func Test_reconcileOrphanedSpecs_ReleasesEnginelessOrphansOnly(t *testing.T) { t.Parallel() liveID := wfTypes.WorkflowID{1} orphanID := wfTypes.WorkflowID{2} @@ -2044,17 +2063,36 @@ func Test_reconcileOrphanedSpecs_DispatchesDeleteForOrphansOnly(t *testing.T) { {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd"}, {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, }} - w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: NewEngineRegistry()} // liveWorkflowIDs is the union of all sources' metadata this tick; the - // live workflow (present in metadata) is retained, the orphan is swept. + // live workflow (present in metadata) is retained, the orphan is released. live := map[string]struct{}{liveID.Hex(): {}} w.reconcileOrphanedSpecs(t.Context(), live) - handled := h.Handled() - require.Len(t, handled, 1, "exactly the orphan is swept") - assert.Equal(t, WorkflowDeleted, handled[0].Name) - del, ok := handled[0].Data.(WorkflowDeletedEvent) - require.True(t, ok) - assert.Equal(t, orphanID, del.WorkflowID, "only the spec absent from metadata is deleted") + assert.Empty(t, h.Handled(), "orphan sweep must not dispatch events through Handle") + released := h.Released() + require.Len(t, released, 1, "exactly the orphan is released") + assert.Equal(t, orphanID.Hex(), released[0].workflowID) + assert.Equal(t, "aabbccdd", released[0].owner) +} + +// An orphan with a registered engine is skipped: the engine-based deletion +// path handles it (with drain/retry machinery). +func Test_reconcileOrphanedSpecs_SkipsOrphansWithEngines(t *testing.T) { + t.Parallel() + orphanID := wfTypes.WorkflowID{3} + er := NewEngineRegistry() + require.NoError(t, er.Add(orphanID, "test-source", &mockService{})) + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, + }} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: er} + + // orphanID is absent from metadata (orphan), but has a running engine. + live := map[string]struct{}{} + w.reconcileOrphanedSpecs(t.Context(), live) + + assert.Empty(t, h.Handled(), "no events dispatched") + assert.Empty(t, h.Released(), "engine-owned orphan must not be released by the sweep") } diff --git a/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql new file mode 100644 index 00000000000..9758ac3d9e6 --- /dev/null +++ b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE workflow_specs_v2 ADD COLUMN registered_at bigint NOT NULL DEFAULT 0; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN registered_at; From bfb6a4b9c2489d8e478cb68b790dcbb8c9754f18 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 31 Jul 2026 10:37:03 -0400 Subject: [PATCH 14/14] removing redundant WorkflowDONSubscriber; lint --- GNUmakefile | 2 +- core/config/toml/types_test.go | 2 + core/services/workflows/syncer/v2/handler.go | 56 ++++------- .../syncer/v2/handler_metering_test.go | 2 +- .../workflows/syncer/v2/handler_test.go | 20 ++-- core/services/workflows/syncer/v2/helpers.go | 2 + .../services/workflows/syncer/v2/mocks/orm.go | 94 +++++++++---------- .../workflows/syncer/v2/workflow_registry.go | 4 +- .../syncer/v2/workflow_registry_test.go | 5 +- 9 files changed, 88 insertions(+), 99 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 3d7fb03dc1c..a5667cc5c86 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -224,7 +224,7 @@ gomodslocalupdate: gomods ## Run gomod-local-update .PHONY: mockery mockery: $(mockery) ## Install mockery. - GOTOOLCHAIN=go$(shell awk '/^go /{print $$2}' go.mod) go install github.com/vektra/mockery/v2@v2.53.0 + go install github.com/vektra/mockery/v2@v2.53.0 .PHONY: codecgen codecgen: $(codecgen) ## Install codecgen diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index 2766aa1570e..89c19b3f1d4 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -862,6 +862,7 @@ func durationPtr(d time.Duration) *commonconfig.Duration { } func TestMetering_ValidateConfig(t *testing.T) { + t.Parallel() testCases := []struct { name string config *Metering @@ -912,6 +913,7 @@ func TestMetering_ValidateConfig(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + t.Parallel() err := tc.config.ValidateConfig() if tc.expectError { require.Error(t, err) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 1ff2776787e..beb03159a8c 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -19,6 +19,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -34,6 +35,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" "github.com/smartcontractkit/chainlink/v2/core/platform" @@ -107,11 +109,8 @@ type eventHandler struct { billingClient metering.BillingClient resourceManager *resourcemanager.ResourceManager meterIdentity resourcemanager.ResourceIdentity - // resolvedDonID holds the workflow DON id once resolved from the don notifier - // at start (the engine runs on the workflow DON). It is resolved - // asynchronously so node boot is not blocked while waiting for the DON to be - // set, and read on the hot snapshot/emit paths; an atomic keeps that read - // lock-free. Nil until resolved; baseIdentity folds it into meterIdentity. + + // resolvedDonID holds the workflow DON id once set by the registry syncer resolvedDonID atomic.Pointer[string] // rmUnregister removes this handler from the ResourceManager's snapshot // registry; set in start, called in close. Nil until started. @@ -357,7 +356,8 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - tracer: noop.NewTracerProvider().Tracer(""), // default; can override in WithDebugMode + // default, enable via WithDebugMode + tracer: noop.NewTracerProvider().Tracer(""), meterIdentity: resourcemanager.ResourceIdentity{ Service: meterService, ResourcePool: meterResourcePool, @@ -400,38 +400,16 @@ func (h *eventHandler) start(ctx context.Context) error { } // Register returns a func that unregisters. h.rmUnregister = h.resourceManager.Register(h) - h.resolveWorkflowDonID() } return nil } -// resolveWorkflowDonID asynchronously resolves the workflow DON id (the engine -// runs on the workflow DON) and folds it into the metering identity. It -// subscribes to the don notifier and stores the first DON's id, so node boot is -// never blocked waiting for the DON to be set. Until it resolves, emitted records -// carry an empty don_id (the host-injection fallback semantics); once resolved, -// every subsequent record and snapshot carries it. Resolution happens at most -// once per start. -func (h *eventHandler) resolveWorkflowDonID() { - if h.workflowDonSubscriber == nil || h.resolvedDonID.Load() != nil { - return - } - h.eng.Go(func(ctx context.Context) { - ch, unsubscribe, err := h.workflowDonSubscriber.Subscribe(ctx) - if err != nil { - h.lggr.Warnw("failed to subscribe to workflow DON for metering identity; don_id will be empty", "err", err) - return - } - defer unsubscribe() - select { - case <-ctx.Done(): - return - case don := <-ch: - donID := strconv.FormatUint(uint64(don.ID), 10) - h.resolvedDonID.Store(&donID) - h.lggr.Debugw("resolved workflow DON id for metering identity", "donID", donID) - } - }) +// SetWorkflowDon supplies the launcher-resolved workflow DON identity for +// metering. Called by the registry after WaitForDon, before any event is +// dispatched; the value is static for the life of the node. +func (h *eventHandler) SetWorkflowDon(don commoncap.DON) { + donID := strconv.FormatUint(uint64(don.ID), 10) + h.resolvedDonID.Store(&donID) } func (h *eventHandler) close() error { @@ -699,7 +677,7 @@ func (h *eventHandler) workflowRegisteredEvent( } else { spec.Status = status if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { - spec.RegisteredAt = int64(payload.CreatedAt) + spec.RegisteredAt = int64(payload.CreatedAt) //nolint:gosec // G115: CreatedAt is a timestamp that cannot overflow int64 } if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { return fmt.Errorf("failed to update workflow spec: %w", innerErr) @@ -711,7 +689,7 @@ func (h *eventHandler) workflowRegisteredEvent( // Legacy-row convergence: backfill registered_at for pre-migration rows // (at most once per row). Fail-open: a write error is logged, not returned. if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { - spec.RegisteredAt = int64(payload.CreatedAt) + spec.RegisteredAt = int64(payload.CreatedAt) //nolint:gosec // G115: CreatedAt is a timestamp that cannot overflow int64 if _, err := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); err != nil { h.lggr.Warnw("failed to backfill registered_at", "workflowID", spec.WorkflowID, "err", err) } @@ -805,7 +783,7 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, Attributes: payload.Attributes, - RegisteredAt: int64(payload.CreatedAt), + RegisteredAt: int64(payload.CreatedAt), //nolint:gosec // G115: CreatedAt is a timestamp that cannot overflow int64 } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -1056,7 +1034,7 @@ func (h *eventHandler) workflowDeletedEvent( if err != nil { return err } - if err := h.releaseSpecStorage(ctx, workflowID, owner); err != nil { + if err = h.releaseSpecStorage(ctx, workflowID, owner); err != nil { return err } _, err = h.engineRegistry.Pop(payload.WorkflowID) @@ -1120,7 +1098,7 @@ func (h *eventHandler) emitSpecDelta(ctx context.Context, delta int64, workflowI } // baseIdentity returns the handler's metering identity with the workflow DON id -// folded in once it has been resolved from the don notifier. meterIdentity itself +// folded in once it has been set by the registry syncer. meterIdentity itself // is immutable after construction (set via WithIdentity); only the DON id is // learned later, so it is read from an atomic and overlaid here. func (h *eventHandler) baseIdentity() resourcemanager.ResourceIdentity { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go index f58f460c439..b2a98743f20 100644 --- a/core/services/workflows/syncer/v2/handler_metering_test.go +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -595,7 +595,7 @@ func Test_meterRecords_DonIDOnRecordAndSnapshot(t *testing.T) { } h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) - // Simulate a resolved workflow DON id (as resolveWorkflowDonID would store). + // Simulate a resolved workflow DON id (as SetWorkflowDon would store). resolvedDon := "7" h.resolvedDonID.Store(&resolvedDon) diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index fcf1f36c551..53caede229e 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -1967,6 +1967,7 @@ func Test_specStorage_StateMachine(t *testing.T) { wfID := types.WorkflowID(wfIDBytes) hexWorkflow := hex.EncodeToString(binaryData) createdAt := uint64(999) + createdAtI64 := int64(createdAt) makeHandler := func(store *stubWorkflowArtifactsStore) *eventHandler { return newMeteringTestHandler(t, store, nil) @@ -1984,7 +1985,7 @@ func Test_specStorage_StateMachine(t *testing.T) { store := &stubWorkflowArtifactsStore{persistUpserts: true} h := makeHandler(store) require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) - assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) _, ok := h.engineRegistry.Get(wfID) assert.True(t, ok, "engine should be started") assert.Equal(t, int32(1), store.fetchCalls.Load(), "exactly one fetch") @@ -1996,12 +1997,12 @@ func Test_specStorage_StateMachine(t *testing.T) { h := makeHandler(store) require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) - assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) _, ok := h.engineRegistry.Get(wfID) assert.False(t, ok, "engine should be popped") // Redelivery require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) - assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) }) t.Run("tombstone + Activated → fetch, full restore, engine started", func(t *testing.T) { @@ -2012,7 +2013,7 @@ func Test_specStorage_StateMachine(t *testing.T) { require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) fetchBefore := store.fetchCalls.Load() require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) - assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) _, ok := h.engineRegistry.Get(wfID) assert.True(t, ok, "engine should be started") assert.Equal(t, fetchBefore+1, store.fetchCalls.Load(), "exactly one new fetch for restore") @@ -2040,12 +2041,12 @@ func Test_specStorage_StateMachine(t *testing.T) { WorkflowOwner: "aabbccdd", Workflow: hexWorkflow, Config: string(configData), - RegisteredAt: int64(createdAt), + RegisteredAt: createdAtI64, }, } h := makeHandler(store) require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) - assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) }) t.Run("absent + Paused → no-op, no error", func(t *testing.T) { @@ -2079,7 +2080,7 @@ func Test_specStorage_StateMachine(t *testing.T) { } h := makeHandler(store) require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) - assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) }) } @@ -2096,6 +2097,7 @@ func Test_handler_SourceParity_PauseActivateCycle(t *testing.T) { require.NoError(t, err) wfID := types.WorkflowID(wfIDBytes) createdAt := uint64(777) + createdAtI64 := int64(createdAt) sources := []string{ "contract:42:0xabcdef1234567890", "grpc:centralized-registry:v1", @@ -2124,10 +2126,10 @@ func Test_handler_SourceParity_PauseActivateCycle(t *testing.T) { assert.Equal(t, source, entry.Source) require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID, Source: source})) - assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) - assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, int64(createdAt)) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) entry, ok = h.engineRegistry.Get(wfID) require.True(t, ok) assert.Equal(t, source, entry.Source) diff --git a/core/services/workflows/syncer/v2/helpers.go b/core/services/workflows/syncer/v2/helpers.go index cefb2b220fb..5ac0780f8a3 100644 --- a/core/services/workflows/syncer/v2/helpers.go +++ b/core/services/workflows/syncer/v2/helpers.go @@ -87,6 +87,8 @@ func (m *testEvtHandler) ReleaseOrphanedSpec(context.Context, string, string) er return nil } +func (m *testEvtHandler) SetWorkflowDon(capabilities.DON) {} + func (m *testEvtHandler) ClearEvents() { m.mux.Lock() defer m.mux.Unlock() diff --git a/core/services/workflows/syncer/v2/mocks/orm.go b/core/services/workflows/syncer/v2/mocks/orm.go index 5a6e40bdbfb..e9c930b3405 100644 --- a/core/services/workflows/syncer/v2/mocks/orm.go +++ b/core/services/workflows/syncer/v2/mocks/orm.go @@ -81,53 +81,6 @@ func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, st return _c } -// PauseWorkflowSpec provides a mock function with given fields: ctx, id -func (_m *ORM) PauseWorkflowSpec(ctx context.Context, id string) error { - ret := _m.Called(ctx, id) - - if len(ret) == 0 { - panic("no return value specified for PauseWorkflowSpec") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(ctx, id) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ORM_PauseWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PauseWorkflowSpec' -type ORM_PauseWorkflowSpec_Call struct { - *mock.Call -} - -// PauseWorkflowSpec is a helper method to define mock.On call -// - ctx context.Context -// - id string -func (_e *ORM_Expecter) PauseWorkflowSpec(ctx interface{}, id interface{}) *ORM_PauseWorkflowSpec_Call { - return &ORM_PauseWorkflowSpec_Call{Call: _e.mock.On("PauseWorkflowSpec", ctx, id)} -} - -func (_c *ORM_PauseWorkflowSpec_Call) Run(run func(ctx context.Context, id string)) *ORM_PauseWorkflowSpec_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string)) - }) - return _c -} - -func (_c *ORM_PauseWorkflowSpec_Call) Return(_a0 error) *ORM_PauseWorkflowSpec_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *ORM_PauseWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_PauseWorkflowSpec_Call { - _c.Call.Return(run) - return _c -} - // DeleteWorkflowSpecs provides a mock function with given fields: ctx, ids func (_m *ORM) DeleteWorkflowSpecs(ctx context.Context, ids []string) error { ret := _m.Called(ctx, ids) @@ -292,6 +245,53 @@ func (_c *ORM_GetWorkflowSpecList_Call) RunAndReturn(run func(context.Context) ( return _c } +// PauseWorkflowSpec provides a mock function with given fields: ctx, id +func (_m *ORM) PauseWorkflowSpec(ctx context.Context, id string) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for PauseWorkflowSpec") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ORM_PauseWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PauseWorkflowSpec' +type ORM_PauseWorkflowSpec_Call struct { + *mock.Call +} + +// PauseWorkflowSpec is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *ORM_Expecter) PauseWorkflowSpec(ctx interface{}, id interface{}) *ORM_PauseWorkflowSpec_Call { + return &ORM_PauseWorkflowSpec_Call{Call: _e.mock.On("PauseWorkflowSpec", ctx, id)} +} + +func (_c *ORM_PauseWorkflowSpec_Call) Run(run func(ctx context.Context, id string)) *ORM_PauseWorkflowSpec_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) Return(_a0 error) *ORM_PauseWorkflowSpec_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_PauseWorkflowSpec_Call { + _c.Call.Return(run) + return _c +} + // UpsertWorkflowSpec provides a mock function with given fields: ctx, spec func (_m *ORM) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { ret := _m.Called(ctx, spec) diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index ffb5af4c8cc..0432cce3d76 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -146,6 +146,7 @@ type evtHandler interface { EmitActivationAbandoned(ctx context.Context, event Event, reason eventsv2.ActivationAbandonReason, activationErr error, retryCount int32) error GetWorkflowSpecList(ctx context.Context) ([]*job.WorkflowSpec, error) ReleaseOrphanedSpec(ctx context.Context, workflowID, owner string) error + SetWorkflowDon(don capabilities.DON) } type donNotifier interface { @@ -422,11 +423,12 @@ func (w *workflowRegistry) Start(_ context.Context) error { return } w.lggr.Debugw("read from don received channel while waiting to start reconciliation sync") - _, err := w.workflowDonNotifier.WaitForDon(ctx) + don, err := w.workflowDonNotifier.WaitForDon(ctx) if err != nil { w.hooks.OnStartFailure(fmt.Errorf("failed to start workflow sync strategy: %w", err)) return } + w.handler.SetWorkflowDon(don) w.syncUsingReconciliationStrategy(ctx) }) diff --git a/core/services/workflows/syncer/v2/workflow_registry_test.go b/core/services/workflows/syncer/v2/workflow_registry_test.go index 9317b3c1a57..d0c3b9ba99c 100644 --- a/core/services/workflows/syncer/v2/workflow_registry_test.go +++ b/core/services/workflows/syncer/v2/workflow_registry_test.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/job" @@ -2033,9 +2034,11 @@ func (h *orphanSweepFakeHandler) GetWorkflowSpecList(context.Context) ([]*job.Wo func (h *orphanSweepFakeHandler) ReleaseOrphanedSpec(_ context.Context, workflowID, owner string) error { h.mu.Lock() defer h.mu.Unlock() - h.released = append(h.released, orphanRelease{workflowID: workflowID, owner: owner}) + h.released = append(h.released, orphanRelease{workflowID, owner}) return nil } + +func (h *orphanSweepFakeHandler) SetWorkflowDon(commonCap.DON) {} func (h *orphanSweepFakeHandler) Handled() []Event { h.mu.Lock() defer h.mu.Unlock()