diff --git a/packages/agentvault/cache.go b/packages/agentvault/cache.go index 47da4419..c07c74df 100644 --- a/packages/agentvault/cache.go +++ b/packages/agentvault/cache.go @@ -225,6 +225,25 @@ func isProxyTokenRejected(err error) bool { return errors.As(err, &apiErr) && apiErr.Name == proxyTokenRejectedName } +// The name the server puts on a refusal because the org used its free Agent Vault requests for the month. Not a +// verdict on the session or the proxy: the proxy keeps running and brokers again once the org adds a payment method or the +// month resets. +const freeLimitReachedName = "AgentVaultFreeLimitReached" + +func isFreeLimitReached(err error) bool { + var apiErr *api.APIError + return errors.As(err, &apiErr) && apiErr.Name == freeLimitReachedName +} + +// The server's message names the limit and the day it resets, so the agent gets it when there is one. +func freeLimitReachedBody(err error) string { + var apiErr *api.APIError + if errors.As(err, &apiErr) && apiErr.ErrorMessage != "" { + return apiErr.ErrorMessage + } + return freeLimitReachedRefusal +} + // The names on Infisical's own 404s and 401s. A route miss, a middlebox or an auth proxy in front of Infisical // answers under another name or none. const ( @@ -307,7 +326,7 @@ func (c *sessionCache) lookup(sessionToken string, admit func() bool) (cacheLook if err != nil { // A rejected proxy token is remembered too: the poll loop exits after two such heartbeats, but // until then every agent request would otherwise cost a resolve. - if isSessionGone(err) || isProxyTokenRejected(err) { + if isSessionGone(err) || isProxyTokenRejected(err) || isFreeLimitReached(err) { c.mu.Lock() c.rememberRefusalLocked(key, err) c.forgetLocked(key, true) @@ -508,7 +527,7 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { entry, ok := c.entries[key] if !ok { // A pinned session whose entry is gone is still polled, and a verdict on it still ends its connections. - if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err)) { + if c.pinnedLocked(key) && (isProxyTokenRejected(err) || isSessionGone(err) || isFreeLimitReached(err)) { c.forgetLocked(key, true) } return @@ -526,6 +545,13 @@ func (c *sessionCache) handleRefreshFailure(key string, err error) { c.forgetLocked(key, true) return } + // Dropped at once, not after the grace window: a credential must not outlive the org's free requests. + if isFreeLimitReached(err) { + log.Warn().Str("sessionId", entry.sessionID).Msg("agent-vault: this organization used its free Agent Vault requests for this month, dropping the session's credentials") + c.rememberRefusalLocked(key, err) + c.forgetLocked(key, true) + return + } // Infisical being unreachable isn't a verdict on the session, so open connections stay and keep being polled // until the grace window has passed since its last answer, when refresh closes them. @@ -579,14 +605,14 @@ func (e *refreshNotRecoveredError) Error() string { return fmt.Sprintf("the refresh did not produce a token the upstream accepts (status %q)", e.status) } -// The refusals that end the session itself. A 404 from credential-refresh can mean the service is gone +// The refusals that drop the session's credentials. A 404 from credential-refresh can mean the service is gone // rather than the session, so it stays a credential failure. func refreshEndedSession(err error) bool { var apiErr *api.APIError if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { return false } - return isProxyTokenRejected(err) || isSessionGone(err) + return isProxyTokenRejected(err) || isSessionGone(err) || isFreeLimitReached(err) } func credentialHash(cred *credential) string { diff --git a/packages/agentvault/cache_test.go b/packages/agentvault/cache_test.go index 031b3832..24e99c8f 100644 --- a/packages/agentvault/cache_test.go +++ b/packages/agentvault/cache_test.go @@ -248,6 +248,35 @@ func TestARejectedProxyTokenDropsTheSessionButIsNotASessionVerdict(t *testing.T) } } +func TestAFreeLimitRefusalDropsTheSessionAtOnceAndIsNotASessionVerdict(t *testing.T) { + err := &api.APIError{StatusCode: 400, Name: freeLimitReachedName, ErrorMessage: "This organization used its 5,000 free Agent Vault requests this month."} + if isSessionGone(err) || isProxyTokenRejected(err) { + t.Fatal("a free limit refusal was read as the session or the proxy being gone") + } + + resolver := &stubResolver{result: &resolveResult{SessionID: "s1", Services: []*resolvedService{serviceWithSecret("v")}}} + cache := newTestCache(resolver) + if _, e := cache.get("tok"); e != nil { + t.Fatal(e) + } + resolver.err = err + cache.refresh() + if _, e := cache.get("tok"); !isFreeLimitReached(e) { + t.Fatalf("the credential kept flowing after the org used its free requests: %v", e) + } +} + +func TestAFreeLimitRefusalIsNotReResolvedEveryRequest(t *testing.T) { + resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: freeLimitReachedName}} + cache := newTestCache(resolver) + for i := 0; i < 20; i++ { + _, _ = cache.get("agv_tok") + } + if got := resolver.callCount(); got != 1 { + t.Fatalf("20 requests from an org past its free requests made %d resolves, want 1", got) + } +} + func TestAnUnnamed401StillEndsTheSession(t *testing.T) { if !isSessionGone(&api.APIError{StatusCode: 401, Name: "UnauthorizedError", ErrorMessage: "Session revoked"}) { t.Fatal("a plain 401 from an older server must still end the session") diff --git a/packages/agentvault/connect.go b/packages/agentvault/connect.go index ba2dcac4..677a2bd7 100644 --- a/packages/agentvault/connect.go +++ b/packages/agentvault/connect.go @@ -43,6 +43,8 @@ func (e *ProxyRefusedError) Error() string { return e.Message } const sessionInvalidRefusal = "the session is no longer valid, or it belongs to a different Agent Vault project than this proxy" +const freeLimitReachedRefusal = "This organization used its free Agent Vault requests for this month" + // SessionInvalid is true when the proxy refused because the session itself can't be used any more. A service refusal // is the same 403, so the code tells them apart. func (e *ProxyRefusedError) SessionInvalid() bool { diff --git a/packages/agentvault/discovery.go b/packages/agentvault/discovery.go index 7c6e0eab..2a9a4183 100644 --- a/packages/agentvault/discovery.go +++ b/packages/agentvault/discovery.go @@ -283,9 +283,13 @@ const ( errorCodeTLSRequired = "tls_required" errorCodeHookDenied = "hook_denied" errorCodeHookFailed = "hook_failed" + errorCodeFreeLimitReached = "free_limit_reached" ) var proxyErrorHints = map[string]string{ + errorCodeFreeLimitReached: "This is the Agent Vault proxy, not the API: this organization used its free Agent Vault requests " + + "for this month, so every request is refused. Pass the message on to the user: an admin can add a payment method in " + + "Infisical under Usage & Billing, or the user can wait for the monthly reset; nothing in this environment can fix it.", errorCodeSessionInvalid: "This is the Agent Vault proxy, not the API: the session this agent runs on has expired or been revoked, " + "or it belongs to a different Agent Vault project than this proxy, so every request is refused. The user has to start a new " + "session, or use a proxy in the session's project; nothing in this environment can fix it.", diff --git a/packages/agentvault/native_logger.go b/packages/agentvault/native_logger.go index b5ae9e1b..4f5b3bca 100644 --- a/packages/agentvault/native_logger.go +++ b/packages/agentvault/native_logger.go @@ -4,6 +4,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -53,6 +54,10 @@ type nativeSessionLogger struct { svc *resolvedService now func() time.Time + // Billed once per connection, however many queries or commands follow, and only once one arrives: PAM logs + // nothing until it has logged in to the service, so a connection the service turned away costs nothing. + billed atomic.Bool + mu sync.Mutex grant *sessionLogGrant // The command's own row is written when it starts. What is piped into it and what it prints follow in rows @@ -74,7 +79,14 @@ func newNativeSessionLogger(ps *proxyServer, key string, grant *sessionLogGrant, return &nativeSessionLogger{ps: ps, key: key, grant: grant, svc: svc, now: time.Now} } +func (l *nativeSessionLogger) bill() { + if l.billed.CompareAndSwap(false, true) { + l.ps.usage.add(1) + } +} + func (l *nativeSessionLogger) LogEntry(entry session.SessionLogEntry) error { + l.bill() detail := &nativeDetail{} appendCapped(&detail.Statement, entry.Input, nativeLogInputCap, &detail.Truncated) appendCapped(&detail.Result, entry.Output, nativeLogOutputCap, &detail.Truncated) @@ -91,6 +103,7 @@ func (l *nativeSessionLogger) LogEntry(entry session.SessionLogEntry) error { } func (l *nativeSessionLogger) LogSessionEvent(event session.SessionEvent) error { + l.bill() data := string(event.Data) l.mu.Lock() diff --git a/packages/agentvault/native_logger_test.go b/packages/agentvault/native_logger_test.go index 53ee0b1f..96862ce1 100644 --- a/packages/agentvault/native_logger_test.go +++ b/packages/agentvault/native_logger_test.go @@ -624,3 +624,26 @@ func TestASilentClientIsClosedWhenConnectStops(t *testing.T) { t.Fatalf("the silent client was left open after cancel: %v", err) } } + +func TestTheLoggerBillsItsConnectionOnceOnTheFirstEntryOrEvent(t *testing.T) { + for _, first := range []func(*nativeSessionLogger){ + func(l *nativeSessionLogger) { _ = l.LogEntry(session.SessionLogEntry{Input: "SELECT 1"}) }, + func(l *nativeSessionLogger) { + _ = l.LogSessionEvent(session.SessionEvent{ChannelType: session.SessionChannelExec, EventType: session.SessionEventInput, Data: []byte("$ whoami")}) + }, + } { + logger, ps := newLoggerUnderTest(t) + ps.usage = newUsageReporter(nil, nil) + logger.recordOutcome(decisionError, "the service refused the login") + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d for an outcome row", n) + } + first(logger) + _ = logger.LogEntry(session.SessionLogEntry{Input: "SELECT 2"}) + _ = logger.LogSessionEvent(session.SessionEvent{ChannelType: session.SessionChannelShell, EventType: session.SessionEventOutput, Data: []byte("ok")}) + _ = logger.Close() + if n := ps.usage.brokered.Load(); n != 1 { + t.Fatalf("billed %d for one connection, want 1", n) + } + } +} diff --git a/packages/agentvault/native_proxy_test.go b/packages/agentvault/native_proxy_test.go index 5d1ab0c2..44feb99a 100644 --- a/packages/agentvault/native_proxy_test.go +++ b/packages/agentvault/native_proxy_test.go @@ -792,3 +792,65 @@ func TestAHookRefusalOfAConnectionReachesConnectAndTheSessionLog(t *testing.T) { }) } } + +func TestADatabaseConnectionIsBilledOnceOnItsFirstQuery(t *testing.T) { + upstream := startFakePostgres(t, "agent", "secret") + ps, client, _ := newNativeProxy(t, []*resolvedService{nativeServiceAt(serviceKindPostgres, "db", upstream)}) + ps.usage = newUsageReporter(nil, nil) + + tunnel, err := client.OpenTunnel(context.Background(), "db") + if err != nil { + t.Fatal(err) + } + defer tunnel.Close() + _ = tunnel.SetDeadline(time.Now().Add(5 * time.Second)) + + frontend := pgproto3.NewFrontend(tunnel, tunnel) + frontend.Send(&pgproto3.StartupMessage{ProtocolVersion: pgproto3.ProtocolVersionNumber, Parameters: map[string]string{"user": "whoever"}}) + if err := frontend.Flush(); err != nil { + t.Fatal(err) + } + waitFor[*pgproto3.ReadyForQuery](t, frontend) + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d before any query", n) + } + for _, query := range []string{"SELECT 1", "SELECT 2"} { + frontend.Send(&pgproto3.Query{String: query}) + if err := frontend.Flush(); err != nil { + t.Fatal(err) + } + waitFor[*pgproto3.ReadyForQuery](t, frontend) + } + frontend.Send(&pgproto3.Terminate{}) + _ = frontend.Flush() + waitNativeDone(t, ps) + + if n := ps.usage.brokered.Load(); n != 1 { + t.Fatalf("billed %d for one connection with two queries, want 1", n) + } +} + +func TestADatabaseConnectionThatFailsToLogInIsNotBilled(t *testing.T) { + upstream := startFakePostgres(t, "agent", "the-real-password") + ps, client, _ := newNativeProxy(t, []*resolvedService{nativeServiceAt(serviceKindPostgres, "db", upstream)}) + ps.usage = newUsageReporter(nil, nil) + + tunnel, err := client.OpenTunnel(context.Background(), "db") + if err != nil { + t.Fatal(err) + } + defer tunnel.Close() + _ = tunnel.SetDeadline(time.Now().Add(5 * time.Second)) + frontend := pgproto3.NewFrontend(tunnel, tunnel) + frontend.Send(&pgproto3.StartupMessage{ProtocolVersion: pgproto3.ProtocolVersionNumber, Parameters: map[string]string{"user": "whoever"}}) + _ = frontend.Flush() + + if records := nativeRecords(t, ps, 1); len(errorRows(records)) != 1 { + t.Fatalf("records = %+v, want the failed login's error row", records) + } + _ = tunnel.Close() + waitNativeDone(t, ps) + if n := ps.usage.brokered.Load(); n != 0 { + t.Fatalf("billed %d for a connection that never logged in", n) + } +} diff --git a/packages/agentvault/proxy.go b/packages/agentvault/proxy.go index 2f348bfe..46ac1807 100644 --- a/packages/agentvault/proxy.go +++ b/packages/agentvault/proxy.go @@ -94,6 +94,7 @@ type proxyServer struct { ca *caManager cache *sessionCache sessionLogs *sessionLogRecorder + usage *usageReporter transport http.RoundTripper // SOCKS5 traffic that no service names goes out through this one, which refuses private destinations // when it dials. Its own pool, so it never reuses a connection the unchecked transport opened. @@ -114,6 +115,7 @@ type proxyServer struct { sseStreams sseStreamRegistry lateNativeLogs lateNativeLogs + refusalLogs refusalLimiter // Open database and SSH tunnels across every session, held under maxNativeTunnels. nativeTunnels atomic.Int64 } @@ -200,7 +202,7 @@ const proxyRevokedBody = "this proxy's access has been revoked" func (ps *proxyServer) denyAtGate(w http.ResponseWriter, sessionToken, hostname, port string, err error) { refusal := classifyGateRefusal(err) - refusal.event. + ps.limitedRefusalEvent("http", refusal, err). Str("host", net.JoinHostPort(hostname, port)). Str("sessionKey", sessionKey(sessionToken)). Str("decision", refusal.decision). @@ -209,6 +211,54 @@ func (ps *proxyServer) denyAtGate(w http.ResponseWriter, sessionToken, hostname, writeProxyError(w, refusal.status, refusal.decision, refusal.code, refusal.body) } +const gateRefusalWarnEvery = time.Minute + +// An agent that keeps calling on a refused session, or a flood of made-up tokens, would otherwise write a line per +// attempt. The first refusal of each kind in a minute keeps its usual line, with how many were held back since; the +// rest go to debug. +type refusalLimiter struct { + mu sync.Mutex + entries map[string]*refusalLog +} + +type refusalLog struct { + last time.Time + suppressed int +} + +func (l *refusalLimiter) admit(key string, now time.Time) (emit bool, suppressed int) { + l.mu.Lock() + defer l.mu.Unlock() + if l.entries == nil { + l.entries = make(map[string]*refusalLog) + } + entry := l.entries[key] + if entry == nil { + entry = &refusalLog{} + l.entries[key] = entry + } + if now.Sub(entry.last) < gateRefusalWarnEvery { + entry.suppressed++ + return false, 0 + } + suppressed = entry.suppressed + entry.last, entry.suppressed = now, 0 + return true, suppressed +} + +// Keyed by door as well, so a flood on one door never hides the first refusal on another. +func (ps *proxyServer) limitedRefusalEvent(door string, refusal gateRefusal, err error) *zerolog.Event { + emit, suppressed := ps.refusalLogs.admit(door+" "+refusal.msg, time.Now()) + if !emit { + refusal.event.Discard() + return log.Debug().Err(err) + } + if suppressed > 0 { + return refusal.event.Int("suppressedSinceLast", suppressed) + } + return refusal.event +} + type gateRefusal struct { event *zerolog.Event decision string @@ -224,6 +274,10 @@ func classifyGateRefusal(err error) gateRefusal { return gateRefusal{log.Error().Err(err), decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody, "agent-vault: refused, Infisical rejected this proxy's token"} } + if isFreeLimitReached(err) { + return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeFreeLimitReached, freeLimitReachedBody(err), + "agent-vault: refused, this organization used its free Agent Vault requests for this month"} + } if isSessionGone(err) { return gateRefusal{log.Warn(), decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal, "agent-vault: refused, the session is no longer valid, or it belongs to a different Agent Vault project than this proxy"} @@ -271,7 +325,7 @@ func (ps *proxyServer) handleConnect(w http.ResponseWriter, r *http.Request) { // The refusal reaches the agent inside the tunnel, but the operator looks for it here, as for any other door. refusal := classifyGateRefusal(err) refused = &refusal - refusal.event. + ps.limitedRefusalEvent("http", refusal, err). Str("host", net.JoinHostPort(hostname, port)). Str("sessionKey", sessionKey(sessionToken)). Str("decision", refusal.decision). @@ -703,6 +757,8 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodePrivateDestination, errPrivateDestination.Error() case isProxyTokenRejected(err): decision, status, code, body = decisionError, http.StatusServiceUnavailable, errorCodeProxyRevoked, proxyRevokedBody + case isFreeLimitReached(err): + decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeFreeLimitReached, freeLimitReachedBody(err) case isSessionGone(err): decision, status, code, body = decisionBlocked, http.StatusForbidden, errorCodeSessionInvalid, sessionInvalidRefusal case errors.Is(err, errSessionResolve): @@ -714,6 +770,9 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun default: status = resp.StatusCode } + if decision == decisionBrokered { + ps.usage.add(1) + } event := log.Debug() switch decision { @@ -724,6 +783,15 @@ func (ps *proxyServer) forwardHTTP(w http.ResponseWriter, r *http.Request, t tun case decisionError: event = log.Error() } + // An agent on a refused session is refused again on every request, so these lines are limited like the gate's. + if !t.viaSocks5 && (refusesSession(code) || code == errorCodeFreeLimitReached) { + if emit, suppressed := ps.refusalLogs.admit("http request "+code, time.Now()); !emit { + event.Discard() + event = log.Debug() + } else if suppressed > 0 { + event = event.Int("suppressedSinceLast", suppressed) + } + } event.Str("method", reqMethod). Str("host", hostname). Str("path", reqPath). diff --git a/packages/agentvault/proxy_gate_test.go b/packages/agentvault/proxy_gate_test.go index c5e1d21a..6370ac58 100644 --- a/packages/agentvault/proxy_gate_test.go +++ b/packages/agentvault/proxy_gate_test.go @@ -4,7 +4,10 @@ import ( "bytes" "encoding/json" "errors" + "net/http" "net/http/httptest" + "net/url" + "strings" "testing" "github.com/Infisical/infisical-merge/packages/api" @@ -63,3 +66,46 @@ func TestGateDenialsAreLogged(t *testing.T) { }) } } + +func TestTheHTTPDoorAtTheFreeLimitPassesOnTheServersMessageAndLogsItOnceAMinute(t *testing.T) { + logs := captureLogs(t) + const serverMessage = "This organization used its 5,000 free Agent Vault requests this month. They reset on December 1." + resolver := &stubResolver{err: &api.APIError{StatusCode: 400, Name: freeLimitReachedName, ErrorMessage: serverMessage}} + ps := &proxyServer{transport: newUpstreamTransport()} + ps.setConfig(ProxyConfig{TrafficPolicy: TrafficPolicyAnyHost}) + ps.cache = newSessionCache(resolver, ps.pollInterval) + + front := httptest.NewServer(http.HandlerFunc(ps.dispatch)) + t.Cleanup(front.Close) + proxyURL, _ := url.Parse(front.URL) + proxyURL.User = url.UserPassword(ProxyAuthUsername, "agv_tok") + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} + + for i := 0; i < 3; i++ { + resp, err := client.Get("http://api.example.com/v1/things") + if err != nil { + t.Fatal(err) + } + var body proxyErrorBody + decodeErr := json.NewDecoder(resp.Body).Decode(&body) + resp.Body.Close() + if decodeErr != nil { + t.Fatal(decodeErr) + } + if resp.StatusCode != http.StatusForbidden || resp.Header.Get(HeaderError) != errorCodeFreeLimitReached { + t.Fatalf("got %d %s=%q, want 403 %s", resp.StatusCode, HeaderError, resp.Header.Get(HeaderError), errorCodeFreeLimitReached) + } + if body.Error != errorCodeFreeLimitReached || body.Message != serverMessage || body.Hint != proxyErrorHints[errorCodeFreeLimitReached] { + t.Fatalf("body = %+v, want the code, the server's message and the hint", body) + } + } + + out := logs.String() + const refusedLine = "this organization used its free Agent Vault requests for this month" + if n := strings.Count(out, `"level":"warn"`); n != 1 { + t.Fatalf("got %d warn lines for 3 refusals in a minute, want 1: %s", n, out) + } + if n := strings.Count(out, refusedLine); n != 3 { + t.Fatalf("got %d refusal lines, want each one at warn or debug: %s", n, out) + } +} diff --git a/packages/agentvault/proxy_policy_test.go b/packages/agentvault/proxy_policy_test.go index c9af44f1..a08ec967 100644 --- a/packages/agentvault/proxy_policy_test.go +++ b/packages/agentvault/proxy_policy_test.go @@ -526,6 +526,7 @@ func newHookedPolicyFixture(t *testing.T, fake *fakeHook) (*proxyServer, *http.C ps, client := newTLSProxy(t, upstream, grantingResolver{services: []*resolvedService{svc}}) ps.opts.ProxyID = "proxy-1" ps.sessionLogs = newSessionLogRecorder("proxy-1", &fakeShipper{}) + ps.usage = newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) return ps, client, host, hits } @@ -590,6 +591,9 @@ func TestAnAllowedRequestIsSentWithItsCredentialAndTheHookSawOnlyTheAgentsReques if rec := drainOneRecord(t, ps.sessionLogs); rec.Hook != nil || rec.Decision != decisionBrokered { t.Fatalf("an allowed request's record = %s", recordJSON(t, rec)) } + if got := ps.usage.brokered.Load(); got != 1 { + t.Fatalf("a brokered request counted %d times for billing, want 1", got) + } } func TestAHookDenialReachesTheAgent(t *testing.T) { @@ -609,6 +613,10 @@ func TestAHookDenialReachesTheAgent(t *testing.T) { t.Fatalf("body = %s", body) } + if got := ps.usage.brokered.Load(); got != 0 { + t.Fatalf("a request the hook denied counted %d times for billing, want 0", got) + } + rec := drainOneRecord(t, ps.sessionLogs) if rec.Decision != decisionBlocked || rec.Status != http.StatusForbidden || rec.Hook == nil || rec.Hook.Decision != hookDeny || rec.Hook.Reason != "over budget" || rec.Hook.DeliveryID == "" { diff --git a/packages/agentvault/run.go b/packages/agentvault/run.go index 6fd6079f..ddfcd149 100644 --- a/packages/agentvault/run.go +++ b/packages/agentvault/run.go @@ -393,6 +393,7 @@ func serve(opts Options, s startup) error { return err } ps.sessionLogs = newSessionLogRecorder(state.ProxyID, shipper) + ps.usage = newUsageReporter(sendUsageReports(opts.ProxyToken), sendFinalUsageReports(opts.ProxyToken)) // Port 0 is not "unset": it is the ordinary ask for any free port, so it is never substituted. listenAddr := net.JoinHostPort(opts.ListenHost, strconv.Itoa(opts.Port)) @@ -455,6 +456,7 @@ func serve(opts Options, s startup) error { fatal := make(chan error, 1) go ps.pollLoop(st, stop, fatal) go ps.sessionLogs.run(stop) + go ps.usage.run(stop) serveErr := make(chan error, 1) go func() { serveErr <- front.Serve(limited) }() @@ -522,7 +524,7 @@ func serve(opts Options, s startup) error { close(stop) closeSocks() ps.cache.closeNative(nativeShutdownWait) - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() if errors.Is(err, http.ErrServerClosed) { return nil @@ -539,7 +541,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return fmt.Errorf("the SOCKS5 listener stopped: %w", err) case <-signals: @@ -550,7 +552,7 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return nil case err := <-fatal: @@ -560,12 +562,19 @@ func serve(opts Options, s startup) error { defer cancel() _ = front.Shutdown(ctx) ps.cache.closeNative(nativeShutdownWait) - closeSessionLogs() + ps.closeReporting(closeSessionLogs) ps.cache.close() return err } } +// Session logs go first: they are the customer's record of what the agents did, and the usage report is small and +// has its own short budget, so it never eats theirs. +func (ps *proxyServer) closeReporting(closeSessionLogs func()) { + closeSessionLogs() + ps.usage.close() +} + // How long shutdown waits for open database and SSH connections to finish, so their last rows reach the session log. const nativeShutdownWait = 2 * time.Second diff --git a/packages/agentvault/socks5.go b/packages/agentvault/socks5.go index 2ed2c5e9..7cb4e1df 100644 --- a/packages/agentvault/socks5.go +++ b/packages/agentvault/socks5.go @@ -183,7 +183,6 @@ func writeSocks5Reply(w io.Writer, rep byte) error { const ( socks5StepTimeout = 3 * time.Second socks5MismatchWarnEvery = time.Minute - socks5RefusalWarnEvery = time.Minute ) type socks5Server struct { @@ -204,14 +203,6 @@ type socks5Server struct { lastMismatchWarn atomic.Int64 budget *missBudget - - refusalsMu sync.Mutex - refusals map[string]*refusalLog -} - -type refusalLog struct { - last time.Time - suppressed int } func newSocks5Server(ps *proxyServer, tlsConfig *tls.Config) *socks5Server { @@ -220,7 +211,6 @@ func newSocks5Server(ps *proxyServer, tlsConfig *tls.Config) *socks5Server { tlsConfig: tlsConfig, stepTimeout: socks5StepTimeout, budget: newMissBudget(socks5MissesPerSecond, socks5MissBurst), - refusals: make(map[string]*refusalLog), } s.idle = sync.NewCond(&s.connsMu) return s @@ -389,33 +379,9 @@ func (s *socks5Server) handle(raw net.Conn) { s.ps.serveIntercepted(conn, tunnel{hostname: hostname, port: port, sessionToken: sessionToken, viaSocks5: true}, leaf) } -// A flood of logins with made-up tokens would otherwise write a line each. The first refusal of each kind in a -// minute keeps its usual line, with how many were held back since; the rest go to debug. +// A flood of logins with made-up tokens would otherwise write a line each. func (s *socks5Server) logRefusal(refusal gateRefusal, err error, remote, sessionToken string) { - now := time.Now() - s.refusalsMu.Lock() - entry := s.refusals[refusal.msg] - if entry == nil { - entry = &refusalLog{} - s.refusals[refusal.msg] = entry - } - suppressed, emit := 0, now.Sub(entry.last) >= socks5RefusalWarnEvery - if emit { - suppressed = entry.suppressed - entry.last, entry.suppressed = now, 0 - } else { - entry.suppressed++ - } - s.refusalsMu.Unlock() - - event := refusal.event - if !emit { - event.Discard() - event = log.Debug().Err(err) - } else if suppressed > 0 { - event = event.Int("suppressedSinceLast", suppressed) - } - event. + s.ps.limitedRefusalEvent("socks5", refusal, err). Str("door", "socks5"). Str("remote", remote). Str("sessionKey", sessionKey(sessionToken)). diff --git a/packages/agentvault/upstream_auth_test.go b/packages/agentvault/upstream_auth_test.go index d0314431..7634ed25 100644 --- a/packages/agentvault/upstream_auth_test.go +++ b/packages/agentvault/upstream_auth_test.go @@ -389,6 +389,27 @@ func TestARefreshThatEndsTheSessionIsAnsweredAsARevokedSession(t *testing.T) { } } +func TestARefreshRefusedForTheFreeLimitIsAnsweredWithTheServersMessage(t *testing.T) { + const message = "This organization used its 5,000 free Agent Vault requests this month. An admin can add a payment method in Usage & Billing, or wait for next month." + resolver := &oauthResolver{err: &api.APIError{StatusCode: http.StatusBadRequest, Name: freeLimitReachedName, ErrorMessage: message}} + client, host, up := newOAuthFixture(t, resolver, func(h string) *resolvedService { return oauthMCPService(h, "old-token") }, acceptOnly("new-token")) + + for range 2 { + resp, body := doWith(t, client, http.MethodGet, fmt.Sprintf("https://%s/mcp", host), "", nil) + refusal := decodeProxyError(t, body) + if resp.StatusCode != http.StatusForbidden || refusal.Error != errorCodeFreeLimitReached || refusal.Message != message { + t.Fatalf("got %d: %s, want 403 %s with the server's message", resp.StatusCode, body, errorCodeFreeLimitReached) + } + } + // The second request is refused from the cache, without reaching the upstream or Infisical. + if got := len(up.recorded()); got != 1 { + t.Fatalf("upstream saw %d requests, want 1", got) + } + if got := len(resolver.refreshCalls()); got != 1 { + t.Fatalf("refresh called %d times, want 1", got) + } +} + func TestRefreshEndedSessionLeavesA404AsACredentialFailure(t *testing.T) { for _, tc := range []struct { err error @@ -397,6 +418,7 @@ func TestRefreshEndedSessionLeavesA404AsACredentialFailure(t *testing.T) { {errSessionGone, true}, {&api.APIError{StatusCode: http.StatusUnauthorized, Name: infisicalUnauthorizedName}, true}, {&api.APIError{StatusCode: http.StatusUnauthorized, Name: proxyTokenRejectedName}, true}, + {&api.APIError{StatusCode: http.StatusBadRequest, Name: freeLimitReachedName}, true}, {&api.APIError{StatusCode: http.StatusNotFound, Name: infisicalNotFoundName}, false}, {&api.APIError{StatusCode: http.StatusUnauthorized}, false}, {&refreshNotRecoveredError{status: oauthStatusNeedsReauthorization}, false}, diff --git a/packages/agentvault/usage.go b/packages/agentvault/usage.go new file mode 100644 index 00000000..0cbd35a6 --- /dev/null +++ b/packages/agentvault/usage.go @@ -0,0 +1,187 @@ +package agentvault + +import ( + "context" + "errors" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/util" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +// How often the proxy reports its brokered requests, and how many unsent reports it keeps (a day's worth) while +// Infisical can't be reached. +const ( + usageReportInterval = time.Minute + maxPendingUsageReports = 24 * 60 + // Infisical refuses a report above this, so a busier minute is split into several. + maxBrokeredPerReport = 1_000_000 + // An Infisical without the route may be mid-upgrade, so the proxy asks again after this rather than giving up. + usageRouteMissingPause = time.Hour + // The last report on the way out gets this long in all, so it never holds up the shutdown. + usageFinalTimeout = 5 * time.Second +) + +// A sealed report never changes. Infisical bills each report once by its id and end time, so a resend that carried +// a different count would be dropped, and counts made after sealing go into the next report. +type usageReport struct { + id string + endedAt time.Time + brokered int64 +} + +type usageSender func(ctx context.Context, report usageReport) error + +type usageReporter struct { + brokered atomic.Int64 + + mu sync.Mutex + pending []usageReport + pausedUntil time.Time + + send usageSender + sendFinal usageSender + now func() time.Time +} + +func newUsageReporter(send, sendFinal usageSender) *usageReporter { + return &usageReporter{send: send, sendFinal: sendFinal, now: time.Now} +} + +// Nil-safe, so a proxy built without a reporter (tests) still serves requests. +func (u *usageReporter) add(n int64) { + if u == nil { + return + } + u.brokered.Add(n) +} + +func (u *usageReporter) seal(now time.Time) { + n := u.brokered.Swap(0) + if n <= 0 { + return + } + u.mu.Lock() + defer u.mu.Unlock() + for n > 0 { + part := min(n, maxBrokeredPerReport) + u.pending = append(u.pending, usageReport{id: uuid.NewString(), endedAt: now.UTC(), brokered: part}) + n -= part + } + if over := len(u.pending) - maxPendingUsageReports; over > 0 { + log.Warn().Int("reports", over).Msg("agent-vault: Infisical has been unreachable for a day, dropping the oldest usage reports") + u.pending = append([]usageReport(nil), u.pending[over:]...) + } +} + +// Sends the pending reports oldest first, and stops at the first one that may still succeed later. +func (u *usageReporter) flush(ctx context.Context, send usageSender) { + u.mu.Lock() + if u.now().Before(u.pausedUntil) { + u.mu.Unlock() + return + } + reports := append([]usageReport(nil), u.pending...) + u.mu.Unlock() + + done := map[string]bool{} + for _, report := range reports { + err := send(ctx, report) + if err == nil { + done[report.id] = true + continue + } + var apiErr *api.APIError + if errors.As(err, &apiErr) { + if apiErr.StatusCode == http.StatusNotFound && apiErr.DocumentationURL != "" { + log.Info().Msg("agent-vault: this Infisical doesn't take usage reports yet, trying again in an hour") + u.mu.Lock() + u.pausedUntil = u.now().Add(usageRouteMissingPause) + u.mu.Unlock() + break + } + // Only a refusal Infisical named is about the report itself, such as a clock too far off, so a resend would + // be refused too. An unnamed one came from something in front of Infisical and may pass later. + if apiErr.Name != "" && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && + apiErr.StatusCode != http.StatusTooManyRequests && apiErr.StatusCode != http.StatusUnauthorized && + apiErr.StatusCode != http.StatusRequestTimeout { + log.Warn().Err(err).Int64("brokered", report.brokered).Msg("agent-vault: Infisical refused a usage report, dropping it") + done[report.id] = true + continue + } + } + log.Debug().Err(err).Msg("agent-vault: couldn't send a usage report, keeping it for the next try") + break + } + + if len(done) == 0 { + return + } + u.mu.Lock() + defer u.mu.Unlock() + kept := u.pending[:0] + for _, report := range u.pending { + if !done[report.id] { + kept = append(kept, report) + } + } + u.pending = kept +} + +func (u *usageReporter) run(stop <-chan struct{}) { + ticker := time.NewTicker(usageReportInterval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case now := <-ticker.C: + u.seal(now) + u.flush(context.Background(), u.send) + } + } +} + +// One last report on the way out, without retries. Anything still unsent is lost with the process. +func (u *usageReporter) close() { + u.seal(u.now()) + ctx, cancel := context.WithTimeout(context.Background(), usageFinalTimeout) + defer cancel() + u.flush(ctx, u.sendFinal) +} + +func sendUsageReports(proxyToken func() string) usageSender { + return func(ctx context.Context, report usageReport) error { + httpClient, err := util.GetRestyClientWithCustomHeaders() + if err != nil { + return err + } + httpClient.SetAuthToken(proxyToken()).SetTimeout(controlPlaneTimeout) + return reportUsage(ctx, httpClient, report) + } +} + +func sendFinalUsageReports(proxyToken func() string) usageSender { + return func(ctx context.Context, report usageReport) error { + httpClient, err := util.GetRestyClientWithPolicy(util.RetryPolicy{}) + if err != nil { + return err + } + httpClient.SetAuthToken(proxyToken()).SetTimeout(usageFinalTimeout) + return reportUsage(ctx, httpClient, report) + } +} + +func reportUsage(ctx context.Context, httpClient *resty.Client, report usageReport) error { + return api.CallReportAgentVaultUsage(ctx, httpClient, api.ReportAgentVaultUsageRequest{ + ReportID: report.id, + EndedAt: report.endedAt, + Brokered: report.brokered, + }) +} diff --git a/packages/agentvault/usage_test.go b/packages/agentvault/usage_test.go new file mode 100644 index 00000000..a20a5e5a --- /dev/null +++ b/packages/agentvault/usage_test.go @@ -0,0 +1,209 @@ +package agentvault + +import ( + "context" + "errors" + "net/http" + "sync" + "testing" + "time" + + "github.com/Infisical/infisical-merge/packages/api" +) + +type recordingSender struct { + mu sync.Mutex + sent []usageReport + errs []error +} + +func (r *recordingSender) send(_ context.Context, report usageReport) error { + r.mu.Lock() + defer r.mu.Unlock() + r.sent = append(r.sent, report) + if len(r.errs) == 0 { + return nil + } + err := r.errs[0] + r.errs = r.errs[1:] + return err +} + +func TestUsageSealsTheCountIntoOneReport(t *testing.T) { + sender := &recordingSender{} + reporter := newUsageReporter(sender.send, sender.send) + + reporter.add(3) + reporter.add(2) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + + if len(sender.sent) != 1 || sender.sent[0].brokered != 5 || sender.sent[0].id == "" { + t.Fatalf("want one report of 5 with an id, got %+v", sender.sent) + } + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + if len(sender.sent) != 1 { + t.Fatalf("an empty minute sent a report: %+v", sender.sent) + } +} + +func TestAFailedReportIsResentUnchangedAndNewCountsGoToANewReport(t *testing.T) { + sender := &recordingSender{errs: []error{errors.New("connection reset")}} + reporter := newUsageReporter(sender.send, sender.send) + + reporter.add(5) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + reporter.add(3) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + + if len(sender.sent) != 3 { + t.Fatalf("want the failed report resent and then the new one, got %+v", sender.sent) + } + first, resent, next := sender.sent[0], sender.sent[1], sender.sent[2] + if resent != first { + t.Fatalf("a resend changed the report: first %+v, resent %+v", first, resent) + } + if next.id == first.id || next.brokered != 3 { + t.Fatalf("new counts must go to a new report, got %+v", next) + } + if len(reporter.pending) != 0 { + t.Fatalf("delivered reports are still pending: %+v", reporter.pending) + } +} + +func TestAReportInfisicalRefusesIsDroppedButAnOutageIsKept(t *testing.T) { + sender := &recordingSender{errs: []error{ + &api.APIError{StatusCode: http.StatusBadRequest, Name: "BadRequest"}, + &api.APIError{StatusCode: http.StatusBadGateway}, + }} + reporter := newUsageReporter(sender.send, sender.send) + + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + + if len(reporter.pending) != 1 { + t.Fatalf("want only the report that hit an outage kept, got %+v", reporter.pending) + } +} + +// Something in front of Infisical, such as a WAF or a load balancer, answers without Infisical's error name, and a +// 408 is a timeout whatever sent it, so neither is a verdict on the report. +func TestAnUnnamedRefusalOrATimeoutKeepsTheReport(t *testing.T) { + for _, refusal := range []*api.APIError{ + {StatusCode: http.StatusForbidden}, + {StatusCode: http.StatusRequestTimeout}, + {StatusCode: http.StatusRequestTimeout, Name: "RequestTimeout"}, + } { + sender := &recordingSender{errs: []error{refusal}} + reporter := newUsageReporter(sender.send, sender.send) + + reporter.add(1) + reporter.seal(time.Now()) + reporter.flush(context.Background(), reporter.send) + if len(reporter.pending) != 1 { + t.Fatalf("%d %q: the report was dropped", refusal.StatusCode, refusal.Name) + } + reporter.flush(context.Background(), reporter.send) + if len(sender.sent) != 2 || len(reporter.pending) != 0 { + t.Fatalf("%d %q: want the report resent and delivered, got sent %+v, pending %+v", refusal.StatusCode, refusal.Name, sender.sent, reporter.pending) + } + } +} + +func TestAnInfisicalWithoutTheRouteIsAskedAgainAfterAnHour(t *testing.T) { + sender := &recordingSender{errs: []error{&api.APIError{StatusCode: http.StatusNotFound, DocumentationURL: "https://infisical.com/docs"}}} + reporter := newUsageReporter(sender.send, sender.send) + now := time.Date(2026, 11, 16, 10, 0, 0, 0, time.UTC) + reporter.now = func() time.Time { return now } + + reporter.add(1) + reporter.seal(now) + reporter.flush(context.Background(), reporter.send) + now = now.Add(30 * time.Minute) + reporter.add(2) + reporter.seal(now) + reporter.flush(context.Background(), reporter.send) + + if len(sender.sent) != 1 || len(reporter.pending) != 2 { + t.Fatalf("want one attempt and both reports kept while paused, got sent %+v, pending %+v", sender.sent, reporter.pending) + } + + now = now.Add(31 * time.Minute) + reporter.flush(context.Background(), reporter.send) + if len(sender.sent) != 3 || len(reporter.pending) != 0 { + t.Fatalf("want both reports sent once the hour passed, got sent %+v, pending %+v", sender.sent, reporter.pending) + } + if sender.sent[1].brokered != 1 || sender.sent[2].brokered != 2 { + t.Fatalf("the paused reports changed: %+v", sender.sent) + } +} + +func TestCloseSendsThroughTheFinalSender(t *testing.T) { + steady, final := &recordingSender{}, &recordingSender{} + reporter := newUsageReporter(steady.send, final.send) + + reporter.add(4) + reporter.close() + + if len(steady.sent) != 0 || len(final.sent) != 1 || final.sent[0].brokered != 4 { + t.Fatalf("want the last report on the final sender, got steady %+v, final %+v", steady.sent, final.sent) + } +} + +func TestShutdownClosesSessionLogsBeforeTheUsageReport(t *testing.T) { + var order []string + ps := &proxyServer{usage: newUsageReporter(nil, func(context.Context, usageReport) error { + order = append(order, "usage") + return nil + })} + ps.usage.add(1) + + ps.closeReporting(func() { order = append(order, "session logs") }) + + if len(order) != 2 || order[0] != "session logs" || order[1] != "usage" { + t.Fatalf("shutdown order = %v, want session logs then usage", order) + } +} + +func TestUnsentReportsAreBounded(t *testing.T) { + reporter := newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) + for i := 0; i < maxPendingUsageReports+5; i++ { + reporter.add(1) + reporter.seal(time.Now()) + } + if len(reporter.pending) != maxPendingUsageReports { + t.Fatalf("want %d pending reports, got %d", maxPendingUsageReports, len(reporter.pending)) + } +} + +func TestANilReporterIgnoresCounts(t *testing.T) { + var reporter *usageReporter + reporter.add(1) +} + +func TestAMinuteOverTheReportLimitIsSplit(t *testing.T) { + reporter := newUsageReporter((&recordingSender{}).send, (&recordingSender{}).send) + reporter.add(2*maxBrokeredPerReport + 5) + reporter.seal(time.Now()) + + if len(reporter.pending) != 3 { + t.Fatalf("want 3 reports, got %+v", reporter.pending) + } + var total int64 + for _, report := range reporter.pending { + if report.brokered > maxBrokeredPerReport { + t.Fatalf("a report carries %d, over the limit", report.brokered) + } + total += report.brokered + } + if total != 2*maxBrokeredPerReport+5 { + t.Fatalf("split lost requests: total %d", total) + } +} diff --git a/packages/api/agent_vault.go b/packages/api/agent_vault.go index 59ce65eb..71ed8ede 100644 --- a/packages/api/agent_vault.go +++ b/packages/api/agent_vault.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "strconv" + "time" "github.com/Infisical/infisical-merge/packages/config" "github.com/go-resty/resty/v2" @@ -262,6 +263,29 @@ func CallReportAgentVaultServiceMissingScopes(httpClient *resty.Client, sessionT return nil } +type ReportAgentVaultUsageRequest struct { + ReportID string `json:"reportId"` + EndedAt time.Time `json:"endedAt"` + Brokered int64 `json:"brokered"` +} + +func CallReportAgentVaultUsage(ctx context.Context, httpClient *resty.Client, request ReportAgentVaultUsageRequest) error { + response, err := httpClient. + R(). + SetContext(ctx). + SetHeader("User-Agent", USER_AGENT). + SetBody(request). + Post(fmt.Sprintf("%v/v1/agent-vault/proxy/usage", config.INFISICAL_URL)) + + if err != nil { + return NewGenericRequestError("CallReportAgentVaultUsage", err) + } + if response.IsError() { + return NewAPIErrorWithResponse("CallReportAgentVaultUsage", response, nil) + } + return nil +} + type CreateAgentVaultSessionLogChunkRequest struct { ChunkID string `json:"chunkId"` EndedAt string `json:"endedAt"`