From 37e09b8fe80dfc3d64dbad2389a0bc4e61d3336d Mon Sep 17 00:00:00 2001 From: saif <11242541+saifsmailbox98@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:43:34 +0530 Subject: [PATCH 1/5] feat(agent-vault): report brokered requests for billing and handle a missing plan - Count each brokered request (HTTP and MCP, after any hook allows it) and each database or SSH connection once - Every minute, and once at shutdown, seal the count into a report with its own id and send it to `POST /v1/agent-vault/proxy/usage`. A report that fails is resent unchanged, since Infisical bills each id once; new counts go into the next report. A day of unsent reports is kept in memory - Stop reporting quietly against an Infisical that doesn't have the route - On `AgentVaultNotOnPlan`, drop the session's credentials at once and answer its requests as `blocked` with `plan_required`. The proxy keeps running and brokers again once the org pays --- packages/agentvault/cache.go | 20 ++- packages/agentvault/cache_test.go | 29 +++++ packages/agentvault/connect.go | 2 + packages/agentvault/discovery.go | 4 + packages/agentvault/native.go | 2 + packages/agentvault/proxy.go | 10 ++ packages/agentvault/proxy_policy_test.go | 8 ++ packages/agentvault/run.go | 6 + packages/agentvault/usage.go | 155 +++++++++++++++++++++++ packages/agentvault/usage_test.go | 122 ++++++++++++++++++ packages/api/agent_vault.go | 23 ++++ 11 files changed, 379 insertions(+), 2 deletions(-) create mode 100644 packages/agentvault/usage.go create mode 100644 packages/agentvault/usage_test.go diff --git a/packages/agentvault/cache.go b/packages/agentvault/cache.go index 47da4419..17b8f1d0 100644 --- a/packages/agentvault/cache.go +++ b/packages/agentvault/cache.go @@ -225,6 +225,15 @@ func isProxyTokenRejected(err error) bool { return errors.As(err, &apiErr) && apiErr.Name == proxyTokenRejectedName } +// The name the server puts on a refusal because the org's plan doesn't include Agent Vault. Not a verdict on the +// session or the proxy: the proxy keeps running and brokers again once the org pays. +const notOnPlanName = "AgentVaultNotOnPlan" + +func isNotOnPlan(err error) bool { + var apiErr *api.APIError + return errors.As(err, &apiErr) && apiErr.Name == notOnPlanName +} + // The names on Infisical's own 404s and 401s. A route miss, a middlebox or an auth proxy in front of Infisical // answers under another name or none. const ( @@ -307,7 +316,7 @@ func (c *sessionCache) lookup(sessionToken string, admit func() bool) (cacheLook if err != nil { // A rejected proxy token is remembered too: the poll loop exits after two such heartbeats, but // until then every agent request would otherwise cost a resolve. - if isSessionGone(err) || isProxyTokenRejected(err) { + if isSessionGone(err) || isProxyTokenRejected(err) || isNotOnPlan(err) { c.mu.Lock() c.rememberRefusalLocked(key, err) c.forgetLocked(key, true) @@ -508,7 +517,7 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { entry, ok := c.entries[key] if !ok { // A pinned session whose entry is gone is still polled, and a verdict on it still ends its connections. - if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err)) { + if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err) || isNotOnPlan(err)) { c.forgetLocked(key, true) } return @@ -526,6 +535,13 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { c.forgetLocked(key, true) return } + // Dropped at once, not after the grace window: a credential must not outlive the org's plan. + if isNotOnPlan(err) { + log.Warn().Str("sessionId", entry.sessionID).Msg("agent-vault: Agent Vault isn't on this organization's plan, dropping the session's credentials") + c.rememberRefusalLocked(key, err) + c.forgetLocked(key, true) + return + } // Infisical being unreachable isn't a verdict on the session, so open connections stay and keep being polled // until the grace window has passed since its last answer, when refresh closes them. diff --git a/packages/agentvault/cache_test.go b/packages/agentvault/cache_test.go index 031b3832..f6b948c8 100644 --- a/packages/agentvault/cache_test.go +++ b/packages/agentvault/cache_test.go @@ -248,6 +248,35 @@ func TestARejectedProxyTokenDropsTheSessionButIsNotASessionVerdict(t *testing.T) } } +func TestAPlanRefusalDropsTheSessionAtOnceAndIsNotASessionVerdict(t *testing.T) { + err := &api.APIError{StatusCode: 400, Name: notOnPlanName, ErrorMessage: "Agent Vault isn't on this organization's plan"} + if isSessionGone(err) || isProxyTokenRejected(err) { + t.Fatal("a plan refusal was read as the session or the proxy being gone") + } + + resolver := &stubResolver{result: &resolveResult{SessionID: "s1", Services: []*resolvedService{serviceWithSecret("v")}}} + cache := newTestCache(resolver) + if _, e := cache.get("tok"); e != nil { + t.Fatal(e) + } + resolver.err = err + cache.refresh() + if _, e := cache.get("tok"); !isNotOnPlan(e) { + t.Fatalf("the credential kept flowing after the org's plan stopped including Agent Vault: %v", e) + } +} + +func TestAPlanRefusalIsNotReResolvedEveryRequest(t *testing.T) { + resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: notOnPlanName}} + cache := newTestCache(resolver) + for i := 0; i < 20; i++ { + _, _ = cache.get("agv_tok") + } + if got := resolver.callCount(); got != 1 { + t.Fatalf("20 requests from an org without the plan made %d resolves, want 1", got) + } +} + func TestAnUnnamed401StillEndsTheSession(t *testing.T) { if !isSessionGone(&api.APIError{StatusCode: 401, Name: "UnauthorizedError", ErrorMessage: "Session revoked"}) { t.Fatal("a plain 401 from an older server must still end the session") diff --git a/packages/agentvault/connect.go b/packages/agentvault/connect.go index ba2dcac4..41f39c8b 100644 --- a/packages/agentvault/connect.go +++ b/packages/agentvault/connect.go @@ -43,6 +43,8 @@ func (e *ProxyRefusedError) Error() string { return e.Message } const sessionInvalidRefusal = "the session is no longer valid, or it belongs to a different Agent Vault project than this proxy" +const notOnPlanRefusal = "Agent Vault isn't on this organization's plan" + // SessionInvalid is true when the proxy refused because the session itself can't be used any more. A service refusal // is the same 403, so the code tells them apart. func (e *ProxyRefusedError) SessionInvalid() bool { diff --git a/packages/agentvault/discovery.go b/packages/agentvault/discovery.go index 7c6e0eab..73fb065e 100644 --- a/packages/agentvault/discovery.go +++ b/packages/agentvault/discovery.go @@ -283,9 +283,13 @@ const ( errorCodeTLSRequired = "tls_required" errorCodeHookDenied = "hook_denied" errorCodeHookFailed = "hook_failed" + errorCodeNotOnPlan = "plan_required" ) var proxyErrorHints = map[string]string{ + errorCodeNotOnPlan: "This is the Agent Vault proxy, not the API: Agent Vault isn't on this organization's plan, so every " + + "request is refused. Tell the user an organization admin can add it in Infisical under Billing; nothing in this " + + "environment can fix it.", errorCodeSessionInvalid: "This is the Agent Vault proxy, not the API: the session this agent runs on has expired or been revoked, " + "or it belongs to a different Agent Vault project than this proxy, so every request is refused. The user has to start a new " + "session, or use a proxy in the session's project; nothing in this environment can fix it.", diff --git a/packages/agentvault/native.go b/packages/agentvault/native.go index 724b43ee..58010e38 100644 --- a/packages/agentvault/native.go +++ b/packages/agentvault/native.go @@ -228,6 +228,8 @@ func (ps *proxyServer) handleNativeConnect(w http.ResponseWriter, r *http.Reques if _, err := clientConn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil { return } + // Billed once per connection: the credential is attached once, however many queries or commands follow. + ps.usage.add(1) logger := newNativeSessionLogger(ps, conn.key, grant, svc) defer logger.Close() diff --git a/packages/agentvault/proxy.go b/packages/agentvault/proxy.go index 2f348bfe..6338e2a6 100644 --- a/packages/agentvault/proxy.go +++ b/packages/agentvault/proxy.go @@ -94,6 +94,7 @@ type proxyServer struct { ca *caManager cache *sessionCache sessionLogs *sessionLogRecorder + usage *usageReporter transport http.RoundTripper // SOCKS5 traffic that no service names goes out through this one, which refuses private destinations // when it dials. Its own pool, so it never reuses a connection the unchecked transport opened. @@ -224,6 +225,10 @@ func classifyGateRefusal(err error) gateRefusal { return gateRefusal{log.Error().Err(err), decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody, "agent-vault: refused, Infisical rejected this proxy's token"} } + if isNotOnPlan(err) { + return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeNotOnPlan, notOnPlanRefusal, + "agent-vault: refused, Agent Vault isn't on this organization's plan"} + } if isSessionGone(err) { return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal, "agent-vault: refused, the session is no longer valid, or it belongs to a different Agent Vault project than this proxy"} @@ -703,6 +708,8 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodePrivateDestination, errPrivateDestination.Error() case isProxyTokenRejected(err): decision, status, code, body = decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody + case isNotOnPlan(err): + decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeNotOnPlan, notOnPlanRefusal case isSessionGone(err): decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal case errors.Is(err, errSessionResolve): @@ -714,6 +721,9 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun default: status = resp.StatusCode } + if decision == decisionBrokered { + ps.usage.add(1) + } event := log.Debug() switch decision { diff --git a/packages/agentvault/proxy_policy_test.go b/packages/agentvault/proxy_policy_test.go index c9af44f1..c573802f 100644 --- a/packages/agentvault/proxy_policy_test.go +++ b/packages/agentvault/proxy_policy_test.go @@ -526,6 +526,7 @@ func newHookedPolicyFixture(t *testing.T, fake *fakeHook) (*proxyServer, *http.C ps, client := newTLSProxy(t, upstream, grantingResolver{services: []*resolvedService{svc}}) ps.opts.ProxyID = "proxy-1" ps.sessionLogs = newSessionLogRecorder("proxy-1", &fakeShipper{}) + ps.usage = newUsageReporter((&recordingSender{}).send) return ps, client, host, hits } @@ -590,6 +591,9 @@ func TestAnAllowedRequestIsSentWithItsCredentialAndTheHookSawOnlyTheAgentsReques if rec := drainOneRecord(t, ps.sessionLogs); rec.Hook != nil || rec.Decision != decisionBrokered { t.Fatalf("an allowed request's record = %s", recordJSON(t, rec)) } + if got := ps.usage.brokered.Load(); got != 1 { + t.Fatalf("a brokered request counted %d times for billing, want 1", got) + } } func TestAHookDenialReachesTheAgent(t *testing.T) { @@ -609,6 +613,10 @@ func TestAHookDenialReachesTheAgent(t *testing.T) { t.Fatalf("body = %s", body) } + if got := ps.usage.brokered.Load(); got != 0 { + t.Fatalf("a request the hook denied counted %d times for billing, want 0", got) + } + rec := drainOneRecord(t, ps.sessionLogs) if rec.Decision != decisionBlocked || rec.Status != http.StatusForbidden || rec.Hook == nil || rec.Hook.Decision != hookDeny || rec.Hook.Reason != "over budget" || rec.Hook.DeliveryID == "" { diff --git a/packages/agentvault/run.go b/packages/agentvault/run.go index 6fd6079f..ac4097ee 100644 --- a/packages/agentvault/run.go +++ b/packages/agentvault/run.go @@ -393,6 +393,7 @@ func serve(opts Options, s startup) error { return err } ps.sessionLogs = newSessionLogRecorder(state.ProxyID, shipper) + ps.usage = newUsageReporter(sendUsageReports(opts.ProxyToken)) // Port 0 is not "unset": it is the ordinary ask for any free port, so it is never substituted. listenAddr := net.JoinHostPort(opts.ListenHost, strconv.Itoa(opts.Port)) @@ -455,6 +456,7 @@ func serve(opts Options, s startup) error { fatal := make(chan error, 1) go ps.pollLoop(st, stop, fatal) go ps.sessionLogs.run(stop) + go ps.usage.run(stop) serveErr := make(chan error, 1) go func() { serveErr <- front.Serve(limited) }() @@ -522,6 +524,7 @@ func serve(opts Options, s startup) error { close(stop) closeSocks() ps.cache.closeNative(nativeShutdownWait) + ps.usage.close() closeSessionLogs() ps.cache.close() if errors.Is(err, http.ErrServerClosed) { @@ -539,6 +542,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) + ps.usage.close() closeSessionLogs() ps.cache.close() return fmt.Errorf("the SOCKS5 listener stopped: %w", err) @@ -550,6 +554,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) + ps.usage.close() closeSessionLogs() ps.cache.close() return nil @@ -560,6 +565,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) + ps.usage.close() closeSessionLogs() ps.cache.close() return err diff --git a/packages/agentvault/usage.go b/packages/agentvault/usage.go new file mode 100644 index 00000000..05b4508f --- /dev/null +++ b/packages/agentvault/usage.go @@ -0,0 +1,155 @@ +package agentvault + +import ( + "errors" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/util" + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +// How often the proxy reports its brokered requests, and how many unsent reports it keeps (a day's worth) while +// Infisical can't be reached. +const ( + usageReportInterval = time.Minute + maxPendingUsageReports = 24 * 60 +) + +// A sealed report never changes. Infisical bills each report once by its id and end time, so a resend that carried +// a different count would be dropped, and counts made after sealing go into the next report. +type usageReport struct { + id string + endedAt time.Time + brokered int64 +} + +type usageSender func(report usageReport) error + +type usageReporter struct { + brokered atomic.Int64 + disabled atomic.Bool + + mu sync.Mutex + pending []usageReport + + send usageSender +} + +func newUsageReporter(send usageSender) *usageReporter { + return &usageReporter{send: send} +} + +// Nil-safe, so a proxy built without a reporter (tests) still serves requests. +func (u *usageReporter) add(n int64) { + if u == nil || u.disabled.Load() { + return + } + u.brokered.Add(n) +} + +func (u *usageReporter) seal(now time.Time) { + n := u.brokered.Swap(0) + if n <= 0 { + return + } + u.mu.Lock() + defer u.mu.Unlock() + u.pending = append(u.pending, usageReport{id: uuid.NewString(), endedAt: now.UTC(), brokered: n}) + if over := len(u.pending) - maxPendingUsageReports; over > 0 { + log.Warn().Int("reports", over).Msg("agent-vault: Infisical has been unreachable for a day, dropping the oldest usage reports") + u.pending = append([]usageReport(nil), u.pending[over:]...) + } +} + +// Sends the pending reports oldest first, and stops at the first one that may still succeed later. +func (u *usageReporter) flush() { + if u.disabled.Load() { + return + } + u.mu.Lock() + reports := append([]usageReport(nil), u.pending...) + u.mu.Unlock() + + done := map[string]bool{} + for _, report := range reports { + err := u.send(report) + if err == nil { + done[report.id] = true + continue + } + var apiErr *api.APIError + if errors.As(err, &apiErr) { + // An Infisical too old to have the route: nothing it could ever accept, so stop counting for this run. + if apiErr.StatusCode == http.StatusNotFound && apiErr.DocumentationURL != "" { + log.Info().Msg("agent-vault: this Infisical doesn't take usage reports, not sending them") + u.disabled.Store(true) + u.mu.Lock() + u.pending = nil + u.mu.Unlock() + return + } + // Refused for what it says, such as a clock too far off, so a resend would be refused too. + if apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && apiErr.StatusCode != http.StatusTooManyRequests && + apiErr.StatusCode != http.StatusUnauthorized { + log.Warn().Err(err).Int64("brokered", report.brokered).Msg("agent-vault: Infisical refused a usage report, dropping it") + done[report.id] = true + continue + } + } + log.Debug().Err(err).Msg("agent-vault: couldn't send a usage report, keeping it for the next try") + break + } + + if len(done) == 0 { + return + } + u.mu.Lock() + defer u.mu.Unlock() + kept := u.pending[:0] + for _, report := range u.pending { + if !done[report.id] { + kept = append(kept, report) + } + } + u.pending = kept +} + +func (u *usageReporter) run(stop <-chan struct{}) { + ticker := time.NewTicker(usageReportInterval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case now := <-ticker.C: + u.seal(now) + u.flush() + } + } +} + +// One last report on the way out. Anything still unsent is lost with the process. +func (u *usageReporter) close() { + u.seal(time.Now()) + u.flush() +} + +func sendUsageReports(proxyToken func() string) usageSender { + return func(report usageReport) error { + httpClient, err := util.GetRestyClientWithCustomHeaders() + if err != nil { + return err + } + httpClient.SetAuthToken(proxyToken()).SetTimeout(controlPlaneTimeout) + return api.CallReportAgentVaultUsage(httpClient, api.ReportAgentVaultUsageRequest{ + ReportID: report.id, + EndedAt: report.endedAt, + Brokered: report.brokered, + }) + } +} diff --git a/packages/agentvault/usage_test.go b/packages/agentvault/usage_test.go new file mode 100644 index 00000000..7eb06875 --- /dev/null +++ b/packages/agentvault/usage_test.go @@ -0,0 +1,122 @@ +package agentvault + +import ( + "errors" + "net/http" + "sync" + "testing" + "time" + + "github.com/Infisical/infisical-merge/packages/api" +) + +type recordingSender struct { + mu sync.Mutex + sent []usageReport + errs []error +} + +func (r *recordingSender) send(report usageReport) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, report) + if len(r.errs) == 0 { + return nil + } + err := r.errs[0] + r.errs = r.errs[1:] + return err +} + +func TestUsageSealsTheCountIntoOneReport(t *testing.T) { + sender := &recordingSender{} + reporter := newUsageReporter(sender.send) + + reporter.add(3) + reporter.add(2) + reporter.seal(time.Now()) + reporter.flush() + + if len(sender.sent) != 1 || sender.sent[0].brokered != 5 || sender.sent[0].id == "" { + t.Fatalf("want one report of 5 with an id, got %+v", sender.sent) + } + reporter.seal(time.Now()) + reporter.flush() + if len(sender.sent) != 1 { + t.Fatalf("an empty minute sent a report: %+v", sender.sent) + } +} + +func TestAFailedReportIsResentUnchangedAndNewCountsGoToANewReport(t *testing.T) { + sender := &recordingSender{errs: []error{errors.New("connection reset")}} + reporter := newUsageReporter(sender.send) + + reporter.add(5) + reporter.seal(time.Now()) + reporter.flush() + reporter.add(3) + reporter.seal(time.Now()) + reporter.flush() + + if len(sender.sent) != 3 { + t.Fatalf("want the failed report resent and then the new one, got %+v", sender.sent) + } + first, resent, next := sender.sent[0], sender.sent[1], sender.sent[2] + if resent != first { + t.Fatalf("a resend changed the report: first %+v, resent %+v", first, resent) + } + if next.id == first.id || next.brokered != 3 { + t.Fatalf("new counts must go to a new report, got %+v", next) + } + if len(reporter.pending) != 0 { + t.Fatalf("delivered reports are still pending: %+v", reporter.pending) + } +} + +func TestAReportInfisicalRefusesIsDroppedButAnOutageIsKept(t *testing.T) { + sender := &recordingSender{errs: []error{&api.APIError{StatusCode: http.StatusBadRequest}, &api.APIError{StatusCode: http.StatusBadGateway}}} + reporter := newUsageReporter(sender.send) + + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush() + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush() + + if len(reporter.pending) != 1 { + t.Fatalf("want only the report that hit an outage kept, got %+v", reporter.pending) + } +} + +func TestAnInfisicalWithoutTheRouteStopsReporting(t *testing.T) { + sender := &recordingSender{errs: []error{&api.APIError{StatusCode: http.StatusNotFound, DocumentationURL: "https://infisical.com/docs"}}} + reporter := newUsageReporter(sender.send) + + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush() + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush() + + if len(sender.sent) != 1 || len(reporter.pending) != 0 { + t.Fatalf("want one attempt and nothing kept, got sent %+v, pending %+v", sender.sent, reporter.pending) + } +} + +func TestUnsentReportsAreBounded(t *testing.T) { + reporter := newUsageReporter((&recordingSender{}).send) + for i := 0; i < maxPendingUsageReports+5; i++ { + reporter.add(1) + reporter.seal(time.Now()) + } + if len(reporter.pending) != maxPendingUsageReports { + t.Fatalf("want %d pending reports, got %d", maxPendingUsageReports, len(reporter.pending)) + } +} + +func TestANilReporterIgnoresCounts(t *testing.T) { + var reporter *usageReporter + reporter.add(1) +} diff --git a/packages/api/agent_vault.go b/packages/api/agent_vault.go index 59ce65eb..7bb28de0 100644 --- a/packages/api/agent_vault.go +++ b/packages/api/agent_vault.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "strconv" + "time" "github.com/Infisical/infisical-merge/packages/config" "github.com/go-resty/resty/v2" @@ -262,6 +263,28 @@ func CallReportAgentVaultServiceMissingScopes(httpClient *resty.Client, sessionT return nil } +type ReportAgentVaultUsageRequest struct { + ReportID string `json:"reportId"` + EndedAt time.Time `json:"endedAt"` + Brokered int64 `json:"brokered"` +} + +func CallReportAgentVaultUsage(httpClient *resty.Client, request ReportAgentVaultUsageRequest) error { + response, err := httpClient. + R(). + SetHeader("User-Agent", USER_AGENT). + SetBody(request). + Post(fmt.Sprintf("%v/v1/agent-vault/proxy/usage", config.INFISICAL_URL)) + + if err != nil { + return NewGenericRequestError("CallReportAgentVaultUsage", err) + } + if response.IsError() { + return NewAPIErrorWithResponse("CallReportAgentVaultUsage", response, nil) + } + return nil +} + type CreateAgentVaultSessionLogChunkRequest struct { ChunkID string `json:"chunkId"` EndedAt string `json:"endedAt"` From 2d3088c55747d8c270d4149bc5216f2b657fc628 Mon Sep 17 00:00:00 2001 From: saif <11242541+saifsmailbox98@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:18:47 +0530 Subject: [PATCH 2/5] feat(agent-vault): handle the free limit refusal, including on OAuth refresh - Rename the plan refusal to AgentVaultFreeLimitReached, answered as 403 free_limit_reached - Pass the server's message (limit and reset date) to the agent as the refusal body, with a hint to tell the user an admin can add a card under Usage & Billing or wait for the monthly reset - An OAuth refresh refused for the free limit drops the session's credentials and answers 403, not 502 --- packages/agentvault/cache.go | 34 +++++++++++++++-------- packages/agentvault/cache_test.go | 16 +++++------ packages/agentvault/connect.go | 2 +- packages/agentvault/discovery.go | 8 +++--- packages/agentvault/proxy.go | 10 +++---- packages/agentvault/upstream_auth_test.go | 22 +++++++++++++++ 6 files changed, 62 insertions(+), 30 deletions(-) diff --git a/packages/agentvault/cache.go b/packages/agentvault/cache.go index 17b8f1d0..a59389b8 100644 --- a/packages/agentvault/cache.go +++ b/packages/agentvault/cache.go @@ -225,13 +225,23 @@ func isProxyTokenRejected(err error) bool { return errors.As(err, &apiErr) && apiErr.Name == proxyTokenRejectedName } -// The name the server puts on a refusal because the org's plan doesn't include Agent Vault. Not a verdict on the -// session or the proxy: the proxy keeps running and brokers again once the org pays. -const notOnPlanName = "AgentVaultNotOnPlan" +// The name the server puts on a refusal because the org used its free Agent Vault requests for the month. Not a +// verdict on the session or the proxy: the proxy keeps running and brokers again once the org adds a card or the +// month resets. +const freeLimitReachedName = "AgentVaultFreeLimitReached" -func isNotOnPlan(err error) bool { +func isFreeLimitReached(err error) bool { var apiErr *api.APIError - return errors.As(err, &apiErr) && apiErr.Name == notOnPlanName + return errors.As(err, &apiErr) && apiErr.Name == freeLimitReachedName +} + +// The server's message names the limit and the day it resets, so the agent gets it when there is one. +func freeLimitReachedBody(err error) string { + var apiErr *api.APIError + if errors.As(err, &apiErr) && apiErr.ErrorMessage != "" { + return apiErr.ErrorMessage + } + return freeLimitReachedRefusal } // The names on Infisical's own 404s and 401s. A route miss, a middlebox or an auth proxy in front of Infisical @@ -316,7 +326,7 @@ func (c *sessionCache) lookup(sessionToken string, admit func() bool) (cacheLook if err != nil { // A rejected proxy token is remembered too: the poll loop exits after two such heartbeats, but // until then every agent request would otherwise cost a resolve. - if isSessionGone(err) || isProxyTokenRejected(err) || isNotOnPlan(err) { + if isSessionGone(err) || isProxyTokenRejected(err) || isFreeLimitReached(err) { c.mu.Lock() c.rememberRefusalLocked(key, err) c.forgetLocked(key, true) @@ -517,7 +527,7 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { entry, ok := c.entries[key] if !ok { // A pinned session whose entry is gone is still polled, and a verdict on it still ends its connections. - if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err) || isNotOnPlan(err)) { + if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err) || isFreeLimitReached(err)) { c.forgetLocked(key, true) } return @@ -535,9 +545,9 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { c.forgetLocked(key, true) return } - // Dropped at once, not after the grace window: a credential must not outlive the org's plan. - if isNotOnPlan(err) { - log.Warn().Str("sessionId", entry.sessionID).Msg("agent-vault: Agent Vault isn't on this organization's plan, dropping the session's credentials") + // Dropped at once, not after the grace window: a credential must not outlive the org's free requests. + if isFreeLimitReached(err) { + log.Warn().Str("sessionId", entry.sessionID).Msg("agent-vault: this organization used its free Agent Vault requests for this month, dropping the session's credentials") c.rememberRefusalLocked(key, err) c.forgetLocked(key, true) return @@ -595,14 +605,14 @@ func (e *refreshNotRecoveredError) Error() string { return fmt.Sprintf("the refresh did not produce a token the upstream accepts (status %q)", e.status) } -// The refusals that end the session itself. A 404 from credential-refresh can mean the service is gone +// The refusals that drop the session's credentials. A 404 from credential-refresh can mean the service is gone // rather than the session, so it stays a credential failure. func refreshEndedSession(err error) bool { var apiErr *api.APIError if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { return false } - return isProxyTokenRejected(err) || isSessionGone(err) + return isProxyTokenRejected(err) || isSessionGone(err) || isFreeLimitReached(err) } func credentialHash(cred *credential) string { diff --git a/packages/agentvault/cache_test.go b/packages/agentvault/cache_test.go index f6b948c8..24e99c8f 100644 --- a/packages/agentvault/cache_test.go +++ b/packages/agentvault/cache_test.go @@ -248,10 +248,10 @@ func TestARejectedProxyTokenDropsTheSessionButIsNotASessionVerdict(t *testing.T) } } -func TestAPlanRefusalDropsTheSessionAtOnceAndIsNotASessionVerdict(t *testing.T) { - err := &api.APIError{StatusCode: 400, Name: notOnPlanName, ErrorMessage: "Agent Vault isn't on this organization's plan"} +func TestAFreeLimitRefusalDropsTheSessionAtOnceAndIsNotASessionVerdict(t *testing.T) { + err := &api.APIError{StatusCode: 400, Name: freeLimitReachedName, ErrorMessage: "This organization used its 5,000 free Agent Vault requests this month."} if isSessionGone(err) || isProxyTokenRejected(err) { - t.Fatal("a plan refusal was read as the session or the proxy being gone") + t.Fatal("a free limit refusal was read as the session or the proxy being gone") } resolver := &stubResolver{result: &resolveResult{SessionID: "s1", Services: []*resolvedService{serviceWithSecret("v")}}} @@ -261,19 +261,19 @@ func TestAPlanRefusalDropsTheSessionAtOnceAndIsNotASessionVerdict(t *testing.T) } resolver.err = err cache.refresh() - if _, e := cache.get("tok"); !isNotOnPlan(e) { - t.Fatalf("the credential kept flowing after the org's plan stopped including Agent Vault: %v", e) + if _, e := cache.get("tok"); !isFreeLimitReached(e) { + t.Fatalf("the credential kept flowing after the org used its free requests: %v", e) } } -func TestAPlanRefusalIsNotReResolvedEveryRequest(t *testing.T) { - resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: notOnPlanName}} +func TestAFreeLimitRefusalIsNotReResolvedEveryRequest(t *testing.T) { + resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: freeLimitReachedName}} cache := newTestCache(resolver) for i := 0; i < 20; i++ { _, _ = cache.get("agv_tok") } if got := resolver.callCount(); got != 1 { - t.Fatalf("20 requests from an org without the plan made %d resolves, want 1", got) + t.Fatalf("20 requests from an org past its free requests made %d resolves, want 1", got) } } diff --git a/packages/agentvault/connect.go b/packages/agentvault/connect.go index 41f39c8b..677a2bd7 100644 --- a/packages/agentvault/connect.go +++ b/packages/agentvault/connect.go @@ -43,7 +43,7 @@ func (e *ProxyRefusedError) Error() string { return e.Message } const sessionInvalidRefusal = "the session is no longer valid, or it belongs to a different Agent Vault project than this proxy" -const notOnPlanRefusal = "Agent Vault isn't on this organization's plan" +const freeLimitReachedRefusal = "This organization used its free Agent Vault requests for this month" // SessionInvalid is true when the proxy refused because the session itself can't be used any more. A service refusal // is the same 403, so the code tells them apart. diff --git a/packages/agentvault/discovery.go b/packages/agentvault/discovery.go index 73fb065e..df24c0ce 100644 --- a/packages/agentvault/discovery.go +++ b/packages/agentvault/discovery.go @@ -283,13 +283,13 @@ const ( errorCodeTLSRequired = "tls_required" errorCodeHookDenied = "hook_denied" errorCodeHookFailed = "hook_failed" - errorCodeNotOnPlan = "plan_required" + errorCodeFreeLimitReached = "free_limit_reached" ) var proxyErrorHints = map[string]string{ - errorCodeNotOnPlan: "This is the Agent Vault proxy, not the API: Agent Vault isn't on this organization's plan, so every " + - "request is refused. Tell the user an organization admin can add it in Infisical under Billing; nothing in this " + - "environment can fix it.", + errorCodeFreeLimitReached: "This is the Agent Vault proxy, not the API: this organization used its free Agent Vault requests " + + "for this month, so every request is refused. Pass the message on to the user: an organization admin can add a card in " + + "Infisical under Usage & Billing, or the user can wait for the monthly reset; nothing in this environment can fix it.", errorCodeSessionInvalid: "This is the Agent Vault proxy, not the API: the session this agent runs on has expired or been revoked, " + "or it belongs to a different Agent Vault project than this proxy, so every request is refused. The user has to start a new " + "session, or use a proxy in the session's project; nothing in this environment can fix it.", diff --git a/packages/agentvault/proxy.go b/packages/agentvault/proxy.go index 6338e2a6..cd96f7ac 100644 --- a/packages/agentvault/proxy.go +++ b/packages/agentvault/proxy.go @@ -225,9 +225,9 @@ func classifyGateRefusal(err error) gateRefusal { return gateRefusal{log.Error().Err(err), decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody, "agent-vault: refused, Infisical rejected this proxy's token"} } - if isNotOnPlan(err) { - return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeNotOnPlan, notOnPlanRefusal, - "agent-vault: refused, Agent Vault isn't on this organization's plan"} + if isFreeLimitReached(err) { + return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeFreeLimitReached, freeLimitReachedBody(err), + "agent-vault: refused, this organization used its free Agent Vault requests for this month"} } if isSessionGone(err) { return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal, @@ -708,8 +708,8 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodePrivateDestination, errPrivateDestination.Error() case isProxyTokenRejected(err): decision, status, code, body = decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody - case isNotOnPlan(err): - decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeNotOnPlan, notOnPlanRefusal + case isFreeLimitReached(err): + decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeFreeLimitReached, freeLimitReachedBody(err) case isSessionGone(err): decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal case errors.Is(err, errSessionResolve): diff --git a/packages/agentvault/upstream_auth_test.go b/packages/agentvault/upstream_auth_test.go index d0314431..550eb2cc 100644 --- a/packages/agentvault/upstream_auth_test.go +++ b/packages/agentvault/upstream_auth_test.go @@ -389,6 +389,27 @@ func TestARefreshThatEndsTheSessionIsAnsweredAsARevokedSession(t *testing.T) { } } +func TestARefreshRefusedForTheFreeLimitIsAnsweredWithTheServersMessage(t *testing.T) { + const message = "This organization used its 5,000 free Agent Vault requests this month. An admin with billing access can add a card in Usage & Billing, or the limit resets on Dec 1." + resolver := &oauthResolver{err: &api.APIError{StatusCode: http.StatusBadRequest, Name: freeLimitReachedName, ErrorMessage: message}} + client, host, up := newOAuthFixture(t, resolver, func(h string) *resolvedService { return oauthMCPService(h, "old-token") }, acceptOnly("new-token")) + + for range 2 { + resp, body := doWith(t, client, http.MethodGet, fmt.Sprintf("https://%s/mcp", host), "", nil) + refusal := decodeProxyError(t, body) + if resp.StatusCode != http.StatusForbidden || refusal.Error != errorCodeFreeLimitReached || refusal.Message != message { + t.Fatalf("got %d: %s, want 403 %s with the server's message", resp.StatusCode, body, errorCodeFreeLimitReached) + } + } + // The second request is refused from the cache, without reaching the upstream or Infisical. + if got := len(up.recorded()); got != 1 { + t.Fatalf("upstream saw %d requests, want 1", got) + } + if got := len(resolver.refreshCalls()); got != 1 { + t.Fatalf("refresh called %d times, want 1", got) + } +} + func TestRefreshEndedSessionLeavesA404AsACredentialFailure(t *testing.T) { for _, tc := range []struct { err error @@ -397,6 +418,7 @@ func TestRefreshEndedSessionLeavesA404AsACredentialFailure(t *testing.T) { {errSessionGone, true}, {&api.APIError{StatusCode: http.StatusUnauthorized, Name: infisicalUnauthorizedName}, true}, {&api.APIError{StatusCode: http.StatusUnauthorized, Name: proxyTokenRejectedName}, true}, + {&api.APIError{StatusCode: http.StatusBadRequest, Name: freeLimitReachedName}, true}, {&api.APIError{StatusCode: http.StatusNotFound, Name: infisicalNotFoundName}, false}, {&api.APIError{StatusCode: http.StatusUnauthorized}, false}, {&refreshNotRecoveredError{status: oauthStatusNeedsReauthorization}, false}, From 1768094c3801b07927c93e2325f431e721aaae2f Mon Sep 17 00:00:00 2001 From: saif <11242541+saifsmailbox98@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:11:30 +0530 Subject: [PATCH 3/5] fix(agent-vault): review round 2 on usage reporting and refusals - A route-missing 404 pauses usage reports for an hour instead of turning them off; counts and pending reports are kept - Drop a report only when Infisical named the refusal; keep it on an unnamed 4xx or a 408 - Shutdown sends session logs first, then the last usage report with 5 seconds and no retries - A database or SSH connection is billed on its first query or command, so one the server turned away costs nothing - Gate refusals on the HTTP door log once a minute per kind, sharing the SOCKS5 limiter - The free limit hint says "an admin" --- packages/agentvault/discovery.go | 2 +- packages/agentvault/native.go | 2 - packages/agentvault/native_logger.go | 13 +++ packages/agentvault/native_logger_test.go | 23 +++++ packages/agentvault/native_proxy_test.go | 62 ++++++++++++ packages/agentvault/proxy.go | 62 +++++++++++- packages/agentvault/proxy_gate_test.go | 46 +++++++++ packages/agentvault/proxy_policy_test.go | 2 +- packages/agentvault/run.go | 21 +++-- packages/agentvault/socks5.go | 38 +------- packages/agentvault/usage.go | 86 +++++++++++------ packages/agentvault/usage_test.go | 109 +++++++++++++++++----- packages/api/agent_vault.go | 3 +- 13 files changed, 366 insertions(+), 103 deletions(-) diff --git a/packages/agentvault/discovery.go b/packages/agentvault/discovery.go index df24c0ce..597e3a2a 100644 --- a/packages/agentvault/discovery.go +++ b/packages/agentvault/discovery.go @@ -288,7 +288,7 @@ const ( var proxyErrorHints = map[string]string{ errorCodeFreeLimitReached: "This is the Agent Vault proxy, not the API: this organization used its free Agent Vault requests " + - "for this month, so every request is refused. Pass the message on to the user: an organization admin can add a card in " + + "for this month, so every request is refused. Pass the message on to the user: an admin can add a card in " + "Infisical under Usage & Billing, or the user can wait for the monthly reset; nothing in this environment can fix it.", errorCodeSessionInvalid: "This is the Agent Vault proxy, not the API: the session this agent runs on has expired or been revoked, " + "or it belongs to a different Agent Vault project than this proxy, so every request is refused. The user has to start a new " + diff --git a/packages/agentvault/native.go b/packages/agentvault/native.go index 58010e38..724b43ee 100644 --- a/packages/agentvault/native.go +++ b/packages/agentvault/native.go @@ -228,8 +228,6 @@ func (ps *proxyServer) handleNativeConnect(w http.ResponseWriter, r *http.Reques if _, err := clientConn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil { return } - // Billed once per connection: the credential is attached once, however many queries or commands follow. - ps.usage.add(1) logger := newNativeSessionLogger(ps, conn.key, grant, svc) defer logger.Close() diff --git a/packages/agentvault/native_logger.go b/packages/agentvault/native_logger.go index b5ae9e1b..4f5b3bca 100644 --- a/packages/agentvault/native_logger.go +++ b/packages/agentvault/native_logger.go @@ -4,6 +4,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -53,6 +54,10 @@ type nativeSessionLogger struct { svc *resolvedService now func() time.Time + // Billed once per connection, however many queries or commands follow, and only once one arrives: PAM logs + // nothing until it has logged in to the service, so a connection the service turned away costs nothing. + billed atomic.Bool + mu sync.Mutex grant *sessionLogGrant // The command's own row is written when it starts. What is piped into it and what it prints follow in rows @@ -74,7 +79,14 @@ func newNativeSessionLogger(ps *proxyServer, key string, grant *sessionLogGrant, return &nativeSessionLogger{ps: ps, key: key, grant: grant, svc: svc, now: time.Now} } +func (l *nativeSessionLogger) bill() { + if l.billed.CompareAndSwap(false, true) { + l.ps.usage.add(1) + } +} + func (l *nativeSessionLogger) LogEntry(entry session.SessionLogEntry) error { + l.bill() detail := &nativeDetail{} appendCapped(&detail.Statement, entry.Input, nativeLogInputCap, &detail.Truncated) appendCapped(&detail.Result, entry.Output, nativeLogOutputCap, &detail.Truncated) @@ -91,6 +103,7 @@ func (l *nativeSessionLogger) LogEntry(entry session.SessionLogEntry) error { } func (l *nativeSessionLogger) LogSessionEvent(event session.SessionEvent) error { + l.bill() data := string(event.Data) l.mu.Lock() diff --git a/packages/agentvault/native_logger_test.go b/packages/agentvault/native_logger_test.go index 53ee0b1f..96862ce1 100644 --- a/packages/agentvault/native_logger_test.go +++ b/packages/agentvault/native_logger_test.go @@ -624,3 +624,26 @@ func TestASilentClientIsClosedWhenConnectStops(t *testing.T) { t.Fatalf("the silent client was left open after cancel: %v", err) } } + +func TestTheLoggerBillsItsConnectionOnceOnTheFirstEntryOrEvent(t *testing.T) { + for _, first := range []func(*nativeSessionLogger){ + func(l *nativeSessionLogger) { _ = l.LogEntry(session.SessionLogEntry{Input: "SELECT 1"}) }, + func(l *nativeSessionLogger) { + _ = l.LogSessionEvent(session.SessionEvent{ChannelType: session.SessionChannelExec, EventType: session.SessionEventInput, Data: []byte("$ whoami")}) + }, + } { + logger, ps := newLoggerUnderTest(t) + ps.usage = newUsageReporter(nil, nil) + logger.recordOutcome(decisionError, "the service refused the login") + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d for an outcome row", n) + } + first(logger) + _ = logger.LogEntry(session.SessionLogEntry{Input: "SELECT 2"}) + _ = logger.LogSessionEvent(session.SessionEvent{ChannelType: session.SessionChannelShell, EventType: session.SessionEventOutput, Data: []byte("ok")}) + _ = logger.Close() + if n := ps.usage.brokered.Load(); n != 1 { + t.Fatalf("billed %d for one connection, want 1", n) + } + } +} diff --git a/packages/agentvault/native_proxy_test.go b/packages/agentvault/native_proxy_test.go index 5d1ab0c2..44feb99a 100644 --- a/packages/agentvault/native_proxy_test.go +++ b/packages/agentvault/native_proxy_test.go @@ -792,3 +792,65 @@ func TestAHookRefusalOfAConnectionReachesConnectAndTheSessionLog(t *testing.T) { }) } } + +func TestADatabaseConnectionIsBilledOnceOnItsFirstQuery(t *testing.T) { + upstream := startFakePostgres(t, "agent", "secret") + ps, client, _ := newNativeProxy(t, []*resolvedService{nativeServiceAt(serviceKindPostgres, "db", upstream)}) + ps.usage = newUsageReporter(nil, nil) + + tunnel, err := client.OpenTunnel(context.Background(), "db") + if err != nil { + t.Fatal(err) + } + defer tunnel.Close() + _ = tunnel.SetDeadline(time.Now().Add(5 * time.Second)) + + frontend := pgproto3.NewFrontend(tunnel, tunnel) + frontend.Send(&pgproto3.StartupMessage{ProtocolVersion: pgproto3.ProtocolVersionNumber, Parameters: map[string]string{"user": "whoever"}}) + if err := frontend.Flush(); err != nil { + t.Fatal(err) + } + waitFor[*pgproto3.ReadyForQuery](t, frontend) + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d before any query", n) + } + for _, query := range []string{"SELECT 1", "SELECT 2"} { + frontend.Send(&pgproto3.Query{String: query}) + if err := frontend.Flush(); err != nil { + t.Fatal(err) + } + waitFor[*pgproto3.ReadyForQuery](t, frontend) + } + frontend.Send(&pgproto3.Terminate{}) + _ = frontend.Flush() + waitNativeDone(t, ps) + + if n := ps.usage.brokered.Load(); n != 1 { + t.Fatalf("billed %d for one connection with two queries, want 1", n) + } +} + +func TestADatabaseConnectionThatFailsToLogInIsNotBilled(t *testing.T) { + upstream := startFakePostgres(t, "agent", "the-real-password") + ps, client, _ := newNativeProxy(t, []*resolvedService{nativeServiceAt(serviceKindPostgres, "db", upstream)}) + ps.usage = newUsageReporter(nil, nil) + + tunnel, err := client.OpenTunnel(context.Background(), "db") + if err != nil { + t.Fatal(err) + } + defer tunnel.Close() + _ = tunnel.SetDeadline(time.Now().Add(5 * time.Second)) + frontend := pgproto3.NewFrontend(tunnel, tunnel) + frontend.Send(&pgproto3.StartupMessage{ProtocolVersion: pgproto3.ProtocolVersionNumber, Parameters: map[string]string{"user": "whoever"}}) + _ = frontend.Flush() + + if records := nativeRecords(t, ps, 1); len(errorRows(records)) != 1 { + t.Fatalf("records = %+v, want the failed login's error row", records) + } + _ = tunnel.Close() + waitNativeDone(t, ps) + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d for a connection that never logged in", n) + } +} diff --git a/packages/agentvault/proxy.go b/packages/agentvault/proxy.go index cd96f7ac..46ac1807 100644 --- a/packages/agentvault/proxy.go +++ b/packages/agentvault/proxy.go @@ -115,6 +115,7 @@ type proxyServer struct { sseStreams sseStreamRegistry lateNativeLogs lateNativeLogs + refusalLogs refusalLimiter // Open database and SSH tunnels across every session, held under maxNativeTunnels. nativeTunnels atomic.Int64 } @@ -201,7 +202,7 @@ const proxyRevokedBody = "this proxy's access has been revoked" func (ps *proxyServer) denyAtGate(w http.ResponseWriter, sessionToken, hostname, port string, err error) { refusal := classifyGateRefusal(err) - refusal.event. + ps.limitedRefusalEvent("http", refusal, err). Str("host", net.JoinHostPort(hostname, port)). Str("sessionKey", sessionKey(sessionToken)). Str("decision", refusal.decision). @@ -210,6 +211,54 @@ func (ps *proxyServer) denyAtGate(w http.ResponseWriter, sessionToken, hostname, writeProxyError(w, refusal.status, refusal.decision, refusal.code, refusal.body) } +const gateRefusalWarnEvery = time.Minute + +// An agent that keeps calling on a refused session, or a flood of made-up tokens, would otherwise write a line per +// attempt. The first refusal of each kind in a minute keeps its usual line, with how many were held back since; the +// rest go to debug. +type refusalLimiter struct { + mu sync.Mutex + entries map[string]*refusalLog +} + +type refusalLog struct { + last time.Time + suppressed int +} + +func (l *refusalLimiter) admit(key string, now time.Time) (emit bool, suppressed int) { + l.mu.Lock() + defer l.mu.Unlock() + if l.entries == nil { + l.entries = make(map[string]*refusalLog) + } + entry := l.entries[key] + if entry == nil { + entry = &refusalLog{} + l.entries[key] = entry + } + if now.Sub(entry.last) < gateRefusalWarnEvery { + entry.suppressed++ + return false, 0 + } + suppressed = entry.suppressed + entry.last, entry.suppressed = now, 0 + return true, suppressed +} + +// Keyed by door as well, so a flood on one door never hides the first refusal on another. +func (ps *proxyServer) limitedRefusalEvent(door string, refusal gateRefusal, err error) *zerolog.Event { + emit, suppressed := ps.refusalLogs.admit(door+" "+refusal.msg, time.Now()) + if !emit { + refusal.event.Discard() + return log.Debug().Err(err) + } + if suppressed > 0 { + return refusal.event.Int("suppressedSinceLast", suppressed) + } + return refusal.event +} + type gateRefusal struct { event *zerolog.Event decision string @@ -276,7 +325,7 @@ func (ps *proxyServer) handleConnect(w http.ResponseWriter, r *http.Request) { // The refusal reaches the agent inside the tunnel, but the operator looks for it here, as for any other door. refusal := classifyGateRefusal(err) refused = &refusal - refusal.event. + ps.limitedRefusalEvent("http", refusal, err). Str("host", net.JoinHostPort(hostname, port)). Str("sessionKey", sessionKey(sessionToken)). Str("decision", refusal.decision). @@ -734,6 +783,15 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun case decisionError: event = log.Error() } + // An agent on a refused session is refused again on every request, so these lines are limited like the gate's. + if !t.viaSocks5 && (refusesSession(code) || code == errorCodeFreeLimitReached) { + if emit, suppressed := ps.refusalLogs.admit("http request "+code, time.Now()); !emit { + event.Discard() + event = log.Debug() + } else if suppressed > 0 { + event = event.Int("suppressedSinceLast", suppressed) + } + } event.Str("method", reqMethod). Str("host", hostname). Str("path", reqPath). diff --git a/packages/agentvault/proxy_gate_test.go b/packages/agentvault/proxy_gate_test.go index c5e1d21a..6370ac58 100644 --- a/packages/agentvault/proxy_gate_test.go +++ b/packages/agentvault/proxy_gate_test.go @@ -4,7 +4,10 @@ import ( "bytes" "encoding/json" "errors" + "net/http" "net/http/httptest" + "net/url" + "strings" "testing" "github.com/Infisical/infisical-merge/packages/api" @@ -63,3 +66,46 @@ func TestGateDenialsAreLogged(t *testing.T) { }) } } + +func TestTheHTTPDoorAtTheFreeLimitPassesOnTheServersMessageAndLogsItOnceAMinute(t *testing.T) { + logs := captureLogs(t) + const serverMessage = "This organization used its 5,000 free Agent Vault requests this month. They reset on December 1." + resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: freeLimitReachedName, ErrorMessage: serverMessage}} + ps := &proxyServer{transport: newUpstreamTransport()} + ps.setConfig(ProxyConfig{TrafficPolicy: TrafficPolicyAnyHost}) + ps.cache = newSessionCache(resolver, ps.pollInterval) + + front := httptest.NewServer(http.HandlerFunc(ps.dispatch)) + t.Cleanup(front.Close) + proxyURL, _ := url.Parse(front.URL) + proxyURL.User = url.UserPassword(ProxyAuthUsername, "agv_tok") + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} + + for i := 0; i < 3; i++ { + resp, err := client.Get("http://api.example.com/v1/things") + if err != nil { + t.Fatal(err) + } + var body proxyErrorBody + decodeErr := json.NewDecoder(resp.Body).Decode(&body) + resp.Body.Close() + if decodeErr != nil { + t.Fatal(decodeErr) + } + if resp.StatusCode != http.StatusForbidden || resp.Header.Get(HeaderError) != errorCodeFreeLimitReached { + t.Fatalf("got %d %s=%q, want 403 %s", resp.StatusCode, HeaderError, resp.Header.Get(HeaderError), errorCodeFreeLimitReached) + } + if body.Error != errorCodeFreeLimitReached || body.Message != serverMessage || body.Hint != proxyErrorHints[errorCodeFreeLimitReached] { + t.Fatalf("body = %+v, want the code, the server's message and the hint", body) + } + } + + out := logs.String() + const refusedLine = "this organization used its free Agent Vault requests for this month" + if n := strings.Count(out, `"level":"warn"`); n != 1 { + t.Fatalf("got %d warn lines for 3 refusals in a minute, want 1: %s", n, out) + } + if n := strings.Count(out, refusedLine); n != 3 { + t.Fatalf("got %d refusal lines, want each one at warn or debug: %s", n, out) + } +} diff --git a/packages/agentvault/proxy_policy_test.go b/packages/agentvault/proxy_policy_test.go index c573802f..a08ec967 100644 --- a/packages/agentvault/proxy_policy_test.go +++ b/packages/agentvault/proxy_policy_test.go @@ -526,7 +526,7 @@ func newHookedPolicyFixture(t *testing.T, fake *fakeHook) (*proxyServer, *http.C ps, client := newTLSProxy(t, upstream, grantingResolver{services: []*resolvedService{svc}}) ps.opts.ProxyID = "proxy-1" ps.sessionLogs = newSessionLogRecorder("proxy-1", &fakeShipper{}) - ps.usage = newUsageReporter((&recordingSender{}).send) + ps.usage = newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) return ps, client, host, hits } diff --git a/packages/agentvault/run.go b/packages/agentvault/run.go index ac4097ee..ddfcd149 100644 --- a/packages/agentvault/run.go +++ b/packages/agentvault/run.go @@ -393,7 +393,7 @@ func serve(opts Options, s startup) error { return err } ps.sessionLogs = newSessionLogRecorder(state.ProxyID, shipper) - ps.usage = newUsageReporter(sendUsageReports(opts.ProxyToken)) + ps.usage = newUsageReporter(sendUsageReports(opts.ProxyToken), sendFinalUsageReports(opts.ProxyToken)) // Port 0 is not "unset": it is the ordinary ask for any free port, so it is never substituted. listenAddr := net.JoinHostPort(opts.ListenHost, strconv.Itoa(opts.Port)) @@ -524,8 +524,7 @@ func serve(opts Options, s startup) error { close(stop) closeSocks() ps.cache.closeNative(nativeShutdownWait) - ps.usage.close() - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() if errors.Is(err, http.ErrServerClosed) { return nil @@ -542,8 +541,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - ps.usage.close() - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return fmt.Errorf("the SOCKS5 listener stopped: %w", err) case <-signals: @@ -554,8 +552,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - ps.usage.close() - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return nil case err := <-fatal: @@ -565,13 +562,19 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - ps.usage.close() - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return err } } +// Session logs go first: they are the customer's record of what the agents did, and the usage report is small and +// has its own short budget, so it never eats theirs. +func (ps *proxyServer) closeReporting(closeSessionLogs func()) { + closeSessionLogs() + ps.usage.close() +} + // How long shutdown waits for open database and SSH connections to finish, so their last rows reach the session log. const nativeShutdownWait = 2 * time.Second diff --git a/packages/agentvault/socks5.go b/packages/agentvault/socks5.go index 2ed2c5e9..7cb4e1df 100644 --- a/packages/agentvault/socks5.go +++ b/packages/agentvault/socks5.go @@ -183,7 +183,6 @@ func writeSocks5Reply(w io.Writer, rep byte) error { const ( socks5StepTimeout = 3 * time.Second socks5MismatchWarnEvery = time.Minute - socks5RefusalWarnEvery = time.Minute ) type socks5Server struct { @@ -204,14 +203,6 @@ type socks5Server struct { lastMismatchWarn atomic.Int64 budget *missBudget - - refusalsMu sync.Mutex - refusals map[string]*refusalLog -} - -type refusalLog struct { - last time.Time - suppressed int } func newSocks5Server(ps *proxyServer, tlsConfig *tls.Config) *socks5Server { @@ -220,7 +211,6 @@ func newSocks5Server(ps *proxyServer, tlsConfig *tls.Config) *socks5Server { tlsConfig: tlsConfig, stepTimeout: socks5StepTimeout, budget: newMissBudget(socks5MissesPerSecond, socks5MissBurst), - refusals: make(map[string]*refusalLog), } s.idle = sync.NewCond(&s.connsMu) return s @@ -389,33 +379,9 @@ func (s *socks5Server) handle(raw net.Conn) { s.ps.serveIntercepted(conn, tunnel{hostname: hostname, port: port, sessionToken: sessionToken, viaSocks5: true}, leaf) } -// A flood of logins with made-up tokens would otherwise write a line each. The first refusal of each kind in a -// minute keeps its usual line, with how many were held back since; the rest go to debug. +// A flood of logins with made-up tokens would otherwise write a line each. func (s *socks5Server) logRefusal(refusal gateRefusal, err error, remote, sessionToken string) { - now := time.Now() - s.refusalsMu.Lock() - entry := s.refusals[refusal.msg] - if entry == nil { - entry = &refusalLog{} - s.refusals[refusal.msg] = entry - } - suppressed, emit := 0, now.Sub(entry.last) >= socks5RefusalWarnEvery - if emit { - suppressed = entry.suppressed - entry.last, entry.suppressed = now, 0 - } else { - entry.suppressed++ - } - s.refusalsMu.Unlock() - - event := refusal.event - if !emit { - event.Discard() - event = log.Debug().Err(err) - } else if suppressed > 0 { - event = event.Int("suppressedSinceLast", suppressed) - } - event. + s.ps.limitedRefusalEvent("socks5", refusal, err). Str("door", "socks5"). Str("remote", remote). Str("sessionKey", sessionKey(sessionToken)). diff --git a/packages/agentvault/usage.go b/packages/agentvault/usage.go index 05b4508f..9441fb51 100644 --- a/packages/agentvault/usage.go +++ b/packages/agentvault/usage.go @@ -1,6 +1,7 @@ package agentvault import ( + "context" "errors" "net/http" "sync" @@ -9,6 +10,7 @@ import ( "github.com/Infisical/infisical-merge/packages/api" "github.com/Infisical/infisical-merge/packages/util" + "github.com/go-resty/resty/v2" "github.com/google/uuid" "github.com/rs/zerolog/log" ) @@ -18,6 +20,10 @@ import ( const ( usageReportInterval = time.Minute maxPendingUsageReports = 24 * 60 + // An Infisical without the route may be mid-upgrade, so the proxy asks again after this rather than giving up. + usageRouteMissingPause = time.Hour + // The last report on the way out gets this long in all, so it never holds up the shutdown. + usageFinalTimeout = 5 * time.Second ) // A sealed report never changes. Infisical bills each report once by its id and end time, so a resend that carried @@ -28,25 +34,27 @@ type usageReport struct { brokered int64 } -type usageSender func(report usageReport) error +type usageSender func(ctx context.Context, report usageReport) error type usageReporter struct { brokered atomic.Int64 - disabled atomic.Bool - mu sync.Mutex - pending []usageReport + mu sync.Mutex + pending []usageReport + pausedUntil time.Time - send usageSender + send usageSender + sendFinal usageSender + now func() time.Time } -func newUsageReporter(send usageSender) *usageReporter { - return &usageReporter{send: send} +func newUsageReporter(send, sendFinal usageSender) *usageReporter { + return &usageReporter{send: send, sendFinal: sendFinal, now: time.Now} } // Nil-safe, so a proxy built without a reporter (tests) still serves requests. func (u *usageReporter) add(n int64) { - if u == nil || u.disabled.Load() { + if u == nil { return } u.brokered.Add(n) @@ -67,35 +75,36 @@ func (u *usageReporter) seal(now time.Time) { } // Sends the pending reports oldest first, and stops at the first one that may still succeed later. -func (u *usageReporter) flush() { - if u.disabled.Load() { +func (u *usageReporter) flush(ctx context.Context, send usageSender) { + u.mu.Lock() + if u.now().Before(u.pausedUntil) { + u.mu.Unlock() return } - u.mu.Lock() reports := append([]usageReport(nil), u.pending...) u.mu.Unlock() done := map[string]bool{} for _, report := range reports { - err := u.send(report) + err := send(ctx, report) if err == nil { done[report.id] = true continue } var apiErr *api.APIError if errors.As(err, &apiErr) { - // An Infisical too old to have the route: nothing it could ever accept, so stop counting for this run. if apiErr.StatusCode == http.StatusNotFound && apiErr.DocumentationURL != "" { - log.Info().Msg("agent-vault: this Infisical doesn't take usage reports, not sending them") - u.disabled.Store(true) + log.Info().Msg("agent-vault: this Infisical doesn't take usage reports yet, trying again in an hour") u.mu.Lock() - u.pending = nil + u.pausedUntil = u.now().Add(usageRouteMissingPause) u.mu.Unlock() - return + break } - // Refused for what it says, such as a clock too far off, so a resend would be refused too. - if apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && apiErr.StatusCode != http.StatusTooManyRequests && - apiErr.StatusCode != http.StatusUnauthorized { + // Only a refusal Infisical named is about the report itself, such as a clock too far off, so a resend would + // be refused too. An unnamed one came from something in front of Infisical and may pass later. + if apiErr.Name != "" && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && + apiErr.StatusCode != http.StatusTooManyRequests && apiErr.StatusCode != http.StatusUnauthorized && + apiErr.StatusCode != http.StatusRequestTimeout { log.Warn().Err(err).Int64("brokered", report.brokered).Msg("agent-vault: Infisical refused a usage report, dropping it") done[report.id] = true continue @@ -128,28 +137,45 @@ func (u *usageReporter) run(stop <-chan struct{}) { return case now := <-ticker.C: u.seal(now) - u.flush() + u.flush(context.Background(), u.send) } } } -// One last report on the way out. Anything still unsent is lost with the process. +// One last report on the way out, without retries. Anything still unsent is lost with the process. func (u *usageReporter) close() { - u.seal(time.Now()) - u.flush() + u.seal(u.now()) + ctx, cancel := context.WithTimeout(context.Background(), usageFinalTimeout) + defer cancel() + u.flush(ctx, u.sendFinal) } func sendUsageReports(proxyToken func() string) usageSender { - return func(report usageReport) error { + return func(ctx context.Context, report usageReport) error { httpClient, err := util.GetRestyClientWithCustomHeaders() if err != nil { return err } httpClient.SetAuthToken(proxyToken()).SetTimeout(controlPlaneTimeout) - return api.CallReportAgentVaultUsage(httpClient, api.ReportAgentVaultUsageRequest{ - ReportID: report.id, - EndedAt: report.endedAt, - Brokered: report.brokered, - }) + return reportUsage(ctx, httpClient, report) + } +} + +func sendFinalUsageReports(proxyToken func() string) usageSender { + return func(ctx context.Context, report usageReport) error { + httpClient, err := util.GetRestyClientWithPolicy(util.RetryPolicy{}) + if err != nil { + return err + } + httpClient.SetAuthToken(proxyToken()).SetTimeout(usageFinalTimeout) + return reportUsage(ctx, httpClient, report) } } + +func reportUsage(ctx context.Context, httpClient *resty.Client, report usageReport) error { + return api.CallReportAgentVaultUsage(ctx, httpClient, api.ReportAgentVaultUsageRequest{ + ReportID: report.id, + EndedAt: report.endedAt, + Brokered: report.brokered, + }) +} diff --git a/packages/agentvault/usage_test.go b/packages/agentvault/usage_test.go index 7eb06875..71789b21 100644 --- a/packages/agentvault/usage_test.go +++ b/packages/agentvault/usage_test.go @@ -1,6 +1,7 @@ package agentvault import ( + "context" "errors" "net/http" "sync" @@ -16,7 +17,7 @@ type recordingSender struct { errs []error } -func (r *recordingSender) send(report usageReport) error { +func (r *recordingSender) send(_ context.Context, report usageReport) error { r.mu.Lock() defer r.mu.Unlock() r.sent = append(r.sent, report) @@ -30,18 +31,18 @@ func (r *recordingSender) send(report usageReport) error { func TestUsageSealsTheCountIntoOneReport(t *testing.T) { sender := &recordingSender{} - reporter := newUsageReporter(sender.send) + reporter := newUsageReporter(sender.send, sender.send) reporter.add(3) reporter.add(2) reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) if len(sender.sent) != 1 || sender.sent[0].brokered != 5 || sender.sent[0].id == "" { t.Fatalf("want one report of 5 with an id, got %+v", sender.sent) } reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) if len(sender.sent) != 1 { t.Fatalf("an empty minute sent a report: %+v", sender.sent) } @@ -49,14 +50,14 @@ func TestUsageSealsTheCountIntoOneReport(t *testing.T) { func TestAFailedReportIsResentUnchangedAndNewCountsGoToANewReport(t *testing.T) { sender := &recordingSender{errs: []error{errors.New("connection reset")}} - reporter := newUsageReporter(sender.send) + reporter := newUsageReporter(sender.send, sender.send) reporter.add(5) reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) reporter.add(3) reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) if len(sender.sent) != 3 { t.Fatalf("want the failed report resent and then the new one, got %+v", sender.sent) @@ -74,39 +75,105 @@ func TestAFailedReportIsResentUnchangedAndNewCountsGoToANewReport(t *testing.T) } func TestAReportInfisicalRefusesIsDroppedButAnOutageIsKept(t *testing.T) { - sender := &recordingSender{errs: []error{&api.APIError{StatusCode: http.StatusBadRequest}, &api.APIError{StatusCode: http.StatusBadGateway}}} - reporter := newUsageReporter(sender.send) + sender := &recordingSender{errs: []error{ + &api.APIError{StatusCode: http.StatusBadRequest, Name: "BadRequest"}, + &api.APIError{StatusCode: http.StatusBadGateway}, + }} + reporter := newUsageReporter(sender.send, sender.send) reporter.add(1) reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) reporter.add(1) reporter.seal(time.Now()) - reporter.flush() + reporter.flush(context.Background(), reporter.send) if len(reporter.pending) != 1 { t.Fatalf("want only the report that hit an outage kept, got %+v", reporter.pending) } } -func TestAnInfisicalWithoutTheRouteStopsReporting(t *testing.T) { +// Something in front of Infisical, such as a WAF or a load balancer, answers without Infisical's error name, and a +// 408 is a timeout whatever sent it, so neither is a verdict on the report. +func TestAnUnnamedRefusalOrATimeoutKeepsTheReport(t *testing.T) { + for _, refusal := range []*api.APIError{ + {StatusCode: http.StatusForbidden}, + {StatusCode: http.StatusRequestTimeout}, + {StatusCode: http.StatusRequestTimeout, Name: "RequestTimeout"}, + } { + sender := &recordingSender{errs: []error{refusal}} + reporter := newUsageReporter(sender.send, sender.send) + + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + if len(reporter.pending) != 1 { + t.Fatalf("%d %q: the report was dropped", refusal.StatusCode, refusal.Name) + } + reporter.flush(context.Background(), reporter.send) + if len(sender.sent) != 2 || len(reporter.pending) != 0 { + t.Fatalf("%d %q: want the report resent and delivered, got sent %+v, pending %+v", refusal.StatusCode, refusal.Name, sender.sent, reporter.pending) + } + } +} + +func TestAnInfisicalWithoutTheRouteIsAskedAgainAfterAnHour(t *testing.T) { sender := &recordingSender{errs: []error{&api.APIError{StatusCode: http.StatusNotFound, DocumentationURL: "https://infisical.com/docs"}}} - reporter := newUsageReporter(sender.send) + reporter := newUsageReporter(sender.send, sender.send) + now := time.Date(2026, 11, 16, 10, 0, 0, 0, time.UTC) + reporter.now = func() time.Time { return now } reporter.add(1) - reporter.seal(time.Now()) - reporter.flush() - reporter.add(1) - reporter.seal(time.Now()) - reporter.flush() + reporter.seal(now) + reporter.flush(context.Background(), reporter.send) + now = now.Add(30 * time.Minute) + reporter.add(2) + reporter.seal(now) + reporter.flush(context.Background(), reporter.send) + + if len(sender.sent) != 1 || len(reporter.pending) != 2 { + t.Fatalf("want one attempt and both reports kept while paused, got sent %+v, pending %+v", sender.sent, reporter.pending) + } + + now = now.Add(31 * time.Minute) + reporter.flush(context.Background(), reporter.send) + if len(sender.sent) != 3 || len(reporter.pending) != 0 { + t.Fatalf("want both reports sent once the hour passed, got sent %+v, pending %+v", sender.sent, reporter.pending) + } + if sender.sent[1].brokered != 1 || sender.sent[2].brokered != 2 { + t.Fatalf("the paused reports changed: %+v", sender.sent) + } +} + +func TestCloseSendsThroughTheFinalSender(t *testing.T) { + steady, final := &recordingSender{}, &recordingSender{} + reporter := newUsageReporter(steady.send, final.send) + + reporter.add(4) + reporter.close() + + if len(steady.sent) != 0 || len(final.sent) != 1 || final.sent[0].brokered != 4 { + t.Fatalf("want the last report on the final sender, got steady %+v, final %+v", steady.sent, final.sent) + } +} + +func TestShutdownClosesSessionLogsBeforeTheUsageReport(t *testing.T) { + var order []string + ps := &proxyServer{usage: newUsageReporter(nil, func(context.Context, usageReport) error { + order = append(order, "usage") + return nil + })} + ps.usage.add(1) + + ps.closeReporting(func() { order = append(order, "session logs") }) - if len(sender.sent) != 1 || len(reporter.pending) != 0 { - t.Fatalf("want one attempt and nothing kept, got sent %+v, pending %+v", sender.sent, reporter.pending) + if len(order) != 2 || order[0] != "session logs" || order[1] != "usage" { + t.Fatalf("shutdown order = %v, want session logs then usage", order) } } func TestUnsentReportsAreBounded(t *testing.T) { - reporter := newUsageReporter((&recordingSender{}).send) + reporter := newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) for i := 0; i < maxPendingUsageReports+5; i++ { reporter.add(1) reporter.seal(time.Now()) diff --git a/packages/api/agent_vault.go b/packages/api/agent_vault.go index 7bb28de0..71ed8ede 100644 --- a/packages/api/agent_vault.go +++ b/packages/api/agent_vault.go @@ -269,9 +269,10 @@ type ReportAgentVaultUsageRequest struct { Brokered int64 `json:"brokered"` } -func CallReportAgentVaultUsage(httpClient *resty.Client, request ReportAgentVaultUsageRequest) error { +func CallReportAgentVaultUsage(ctx context.Context, httpClient *resty.Client, request ReportAgentVaultUsageRequest) error { response, err := httpClient. R(). + SetContext(ctx). SetHeader("User-Agent", USER_AGENT). SetBody(request). Post(fmt.Sprintf("%v/v1/agent-vault/proxy/usage", config.INFISICAL_URL)) From 4acce4d7a33a7a3aec89f0841c7befe67f427a44 Mon Sep 17 00:00:00 2001 From: saif <11242541+saifsmailbox98@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:44:49 +0530 Subject: [PATCH 4/5] fix(agent-vault): say payment method instead of card in the free limit hint --- packages/agentvault/cache.go | 2 +- packages/agentvault/discovery.go | 2 +- packages/agentvault/upstream_auth_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/agentvault/cache.go b/packages/agentvault/cache.go index a59389b8..c07c74df 100644 --- a/packages/agentvault/cache.go +++ b/packages/agentvault/cache.go @@ -226,7 +226,7 @@ func isProxyTokenRejected(err error) bool { } // The name the server puts on a refusal because the org used its free Agent Vault requests for the month. Not a -// verdict on the session or the proxy: the proxy keeps running and brokers again once the org adds a card or the +// verdict on the session or the proxy: the proxy keeps running and brokers again once the org adds a payment method or the // month resets. const freeLimitReachedName = "AgentVaultFreeLimitReached" diff --git a/packages/agentvault/discovery.go b/packages/agentvault/discovery.go index 597e3a2a..2a9a4183 100644 --- a/packages/agentvault/discovery.go +++ b/packages/agentvault/discovery.go @@ -288,7 +288,7 @@ const ( var proxyErrorHints = map[string]string{ errorCodeFreeLimitReached: "This is the Agent Vault proxy, not the API: this organization used its free Agent Vault requests " + - "for this month, so every request is refused. Pass the message on to the user: an admin can add a card in " + + "for this month, so every request is refused. Pass the message on to the user: an admin can add a payment method in " + "Infisical under Usage & Billing, or the user can wait for the monthly reset; nothing in this environment can fix it.", errorCodeSessionInvalid: "This is the Agent Vault proxy, not the API: the session this agent runs on has expired or been revoked, " + "or it belongs to a different Agent Vault project than this proxy, so every request is refused. The user has to start a new " + diff --git a/packages/agentvault/upstream_auth_test.go b/packages/agentvault/upstream_auth_test.go index 550eb2cc..7634ed25 100644 --- a/packages/agentvault/upstream_auth_test.go +++ b/packages/agentvault/upstream_auth_test.go @@ -390,7 +390,7 @@ func TestARefreshThatEndsTheSessionIsAnsweredAsARevokedSession(t *testing.T) { } func TestARefreshRefusedForTheFreeLimitIsAnsweredWithTheServersMessage(t *testing.T) { - const message = "This organization used its 5,000 free Agent Vault requests this month. An admin with billing access can add a card in Usage & Billing, or the limit resets on Dec 1." + const message = "This organization used its 5,000 free Agent Vault requests this month. An admin can add a payment method in Usage & Billing, or wait for next month." resolver := &oauthResolver{err: &api.APIError{StatusCode: http.StatusBadRequest, Name: freeLimitReachedName, ErrorMessage: message}} client, host, up := newOAuthFixture(t, resolver, func(h string) *resolvedService { return oauthMCPService(h, "old-token") }, acceptOnly("new-token")) From 09e36795e85f3dcd6284fa4f9378915ef0eaf68e Mon Sep 17 00:00:00 2001 From: saif <11242541+saifsmailbox98@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:59:48 +0530 Subject: [PATCH 5/5] fix(agent-vault): split a minute over 1,000,000 requests into several usage reports --- packages/agentvault/usage.go | 8 +++++++- packages/agentvault/usage_test.go | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/agentvault/usage.go b/packages/agentvault/usage.go index 9441fb51..0cbd35a6 100644 --- a/packages/agentvault/usage.go +++ b/packages/agentvault/usage.go @@ -20,6 +20,8 @@ import ( const ( usageReportInterval = time.Minute maxPendingUsageReports = 24 * 60 + // Infisical refuses a report above this, so a busier minute is split into several. + maxBrokeredPerReport = 1_000_000 // An Infisical without the route may be mid-upgrade, so the proxy asks again after this rather than giving up. usageRouteMissingPause = time.Hour // The last report on the way out gets this long in all, so it never holds up the shutdown. @@ -67,7 +69,11 @@ func (u *usageReporter) seal(now time.Time) { } u.mu.Lock() defer u.mu.Unlock() - u.pending = append(u.pending, usageReport{id: uuid.NewString(), endedAt: now.UTC(), brokered: n}) + for n > 0 { + part := min(n, maxBrokeredPerReport) + u.pending = append(u.pending, usageReport{id: uuid.NewString(), endedAt: now.UTC(), brokered: part}) + n -= part + } if over := len(u.pending) - maxPendingUsageReports; over > 0 { log.Warn().Int("reports", over).Msg("agent-vault: Infisical has been unreachable for a day, dropping the oldest usage reports") u.pending = append([]usageReport(nil), u.pending[over:]...) diff --git a/packages/agentvault/usage_test.go b/packages/agentvault/usage_test.go index 71789b21..a20a5e5a 100644 --- a/packages/agentvault/usage_test.go +++ b/packages/agentvault/usage_test.go @@ -187,3 +187,23 @@ func TestANilReporterIgnoresCounts(t *testing.T) { var reporter *usageReporter reporter.add(1) } + +func TestAMinuteOverTheReportLimitIsSplit(t *testing.T) { + reporter := newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) + reporter.add(2*maxBrokeredPerReport + 5) + reporter.seal(time.Now()) + + if len(reporter.pending) != 3 { + t.Fatalf("want 3 reports, got %+v", reporter.pending) + } + var total int64 + for _, report := range reporter.pending { + if report.brokered > maxBrokeredPerReport { + t.Fatalf("a report carries %d, over the limit", report.brokered) + } + total += report.brokered + } + if total != 2*maxBrokeredPerReport+5 { + t.Fatalf("split lost requests: total %d", total) + } +}