fix(egress-prober): fail loudly when the server rejects the byJwt

cmd/egress-prober/credential_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// The credential check exists to separate three outcomes the prober previously
+// could not tell apart: accepted, refused, and unknown. Only the middle one may
+// stop the process, so each is asserted on its own.
+
+func TestCheckCredentialAcceptsOK(t *testing.T) {
+	var gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	if err := checkCredential(context.Background(), srv.Client(), srv.URL, "the-jwt"); err != nil {
+		t.Fatalf("checkCredential: %s, want nil", err)
+	}
+	// Without the header the endpoint would answer 401 for a reason that has
+	// nothing to do with the credential, and the check would condemn a working
+	// token.
+	if want := "Bearer the-jwt"; gotAuth != want {
+		t.Errorf("Authorization = %q, want %q", gotAuth, want)
+	}
+}
+
+func TestCheckCredentialRejects401(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusUnauthorized)
+	}))
+	defer srv.Close()
+
+	err := checkCredential(context.Background(), srv.Client(), srv.URL, "stale-jwt")
+	if !errors.Is(err, errCredentialRejected) {
+		t.Fatalf("checkCredential = %v, want errCredentialRejected", err)
+	}
+	// A rejection that also satisfied errCredentialUnverified would be
+	// downgraded to a warning by main's switch, which is the whole failure this
+	// change exists to prevent.
+	if errors.Is(err, errCredentialUnverified) {
+		t.Errorf("a rejection also matched errCredentialUnverified; main would let the prober start")
+	}
+}
+
+// A server that predates the endpoint answers 404. That says nothing about the
+// credential, so it must not stop the prober -- the same posture ingest takes
+// when the due endpoint is missing.
+func TestCheckCredentialTreats404AsUnverified(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	err := checkCredential(context.Background(), srv.Client(), srv.URL, "fine-jwt")
+	if !errors.Is(err, errCredentialUnverified) {
+		t.Fatalf("checkCredential = %v, want errCredentialUnverified", err)
+	}
+	if errors.Is(err, errCredentialRejected) {
+		t.Errorf("404 was reported as a rejected credential; an old server would stop the prober")
+	}
+}
+
+func TestCheckCredentialTreatsTransportErrorAsUnverified(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	url := srv.URL
+	srv.Close() // nothing is listening now
+
+	err := checkCredential(context.Background(), http.DefaultClient, url, "fine-jwt")
+	if !errors.Is(err, errCredentialUnverified) {
+		t.Fatalf("checkCredential = %v, want errCredentialUnverified", err)
+	}
+	if errors.Is(err, errCredentialRejected) {
+		t.Errorf("an unreachable server was reported as a rejected credential")
+	}
+}

cmd/egress-prober/main.go

@@ -66,7 +66,7 @@ func main() {
 	operatorSecret := flag.String("operator-secret", "", "ingest secret, must match ingest_secret in provider_egress.yml; prefer the UR_OPERATOR_SECRET env var, which keeps it out of ps (required)")
 	concurrency := flag.Int("concurrency", 4, "max simultaneous provider tunnels")
 	cacheTTL := flag.Duration("cache-ttl", 24*time.Hour, "do not re-probe a provider within this window. Only applies to the enumeration fallback used against a server with no due endpoint; when the server supplies the due list it owns the schedule")
-	interval := flag.Duration("interval", time.Hour, "sleep between passes; 0 runs a single pass and exits")
+	interval := flag.Duration("interval", time.Hour, "sleep AFTER a pass finishes, not a fixed period: the cycle is pass-duration + interval, so throughput is -due-limit / (pass-duration + interval) rather than -due-limit per interval. A 500-provider pass taking ~30m at -interval 1h yields ~390/hour with the prober idle two thirds of every cycle. Size it against how long a pass actually takes; 0 runs a single pass and exits")
 	probeTimeout := flag.Duration("probe-timeout", 60*time.Second, "per-provider probe timeout, and the per-source deadline within a probe")
 	skipConfinementCheck := flag.Bool("skip-confinement-check", false, "DANGEROUS: start even if this host can reach a geolocation api directly. Only for a one-shot manual probe on a host you know is not the operator's; a direct lookup records the OPERATOR's location for the provider and exposes the operator's address to the api")
 	confinementTimeout := flag.Duration("confinement-timeout", 3*time.Second, "per-address deadline for the startup confinement self-check; a timeout counts as blocked. Must be at least "+confinement.MinTimeout.String())
@@ -256,6 +256,21 @@ func main() {
 		log.Fatalf("parse by-jwt client id: %s", err)
 	}
 
+	// The credential self-check runs before the first tunnel, for the same
+	// reason the confinement self-check runs before the first request: a fault
+	// the prober cannot detect at runtime has to be caught here or not at all.
+	switch err := checkCredential(ctx, http.DefaultClient, *apiURL, *byJwt); {
+	case err == nil:
+		log.Printf("egress-prober: credential self-check passed: the server accepts the byJwt")
+	case errors.Is(err, errCredentialRejected):
+		log.Printf("egress-prober: credential self-check FAILED: %s", err)
+		log.Printf("egress-prober: the byJwt parses but the server refuses it -- it has expired, or it predates a claim the server now enforces. Probes would not fail loudly: the operator-secret calls would keep working while every tunnel carried nothing and every provider was recorded as no_consensus. Mint a fresh network client jwt (POST /network/auth-client) and set UR_PROBER_BY_JWT to it.")
+		os.Exit(1)
+	default:
+		log.Printf("egress-prober: WARNING credential self-check inconclusive: %s", err)
+		log.Printf("egress-prober: continuing, because being unable to check is not evidence the credential is bad. If every provider comes back no_consensus with ok=0/N, suspect the byJwt first.")
+	}
+
 	// Pins is deliberately NOT set here: it is fetched from the server below
 	// and read afresh on every tunnel Open, so an hourly refresh reaches the
 	// next provider rather than only the next process.
@@ -1205,6 +1220,63 @@ func findProvidersAtLocation(ctx context.Context, client *http.Client, apiURL st
 // is the authority that already validated it when minting a session from it),
 // and the claim is type-switched rather than unmarshaled into a typed struct,
 // since some issuers emit client_id as something other than a bare string.
+// errCredentialRejected reports that the server refused the prober's byJwt.
+//
+// This is kept distinct from "could not check" for the same reason
+// ingest.ErrUnauthorized is kept distinct from ingest.ErrDueUnsupported: a
+// rejected credential is a broken deployment, and anything that lets it look
+// like an ordinary runtime hiccup hides the fault behind work that appears to
+// continue.
+var errCredentialRejected = errors.New("egress-prober: the server rejected the prober's byJwt")
+
+// errCredentialUnverified reports that the check could not reach a verdict --
+// an old server without the endpoint, a transport error, a 5xx. It is NOT a
+// claim that the credential is bad, so it must never stop the prober: doing so
+// would turn "we could not ask" into a new outage of its own.
+var errCredentialUnverified = errors.New("egress-prober: could not verify the byJwt")
+
+// checkCredential asks the server whether it ACCEPTS the byJwt, which is not
+// what parseByJwtClientId establishes -- that only proves the token decodes and
+// carries a client_id.
+//
+// The gap between those two is a real outage mode, not a hypothetical. A token
+// that parses perfectly is still refused once it expires (jwt.expiryDuration is
+// 24h) or once the server begins enforcing a claim the token predates, and the
+// prober has no way to notice: the byJwt authenticates only the provider
+// tunnel, while the due queue, attempt reporting and pin fetch all authenticate
+// with the operator secret. So every one of those keeps working, the prober
+// goes on reporting attempts and looks healthy, and the tunnel silently carries
+// nothing -- every probe fails no_consensus with ok=0/N.
+//
+// That reads as "the whole fleet is bad", which is a convincing wrong answer.
+// On one deployment it ran 8 hours and 870 consecutive failures before the
+// credential was suspected. One request at startup turns that into a message.
+func checkCredential(ctx context.Context, client *http.Client, apiURL string, byJwt string) error {
+	url := strings.TrimRight(apiURL, "/") + "/network/clients"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errCredentialUnverified, err)
+	}
+	req.Header.Set("Authorization", "Bearer "+byJwt)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errCredentialUnverified, err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return errCredentialRejected
+	default:
+		// Everything else -- notably 404 from a server that predates this
+		// endpoint -- is inconclusive by design.
+		return fmt.Errorf("%w: status %d", errCredentialUnverified, resp.StatusCode)
+	}
+}
+
 func parseByJwtClientId(byJwt string) (connect.Id, error) {
 	claims := gojwt.MapClaims{}
 	if _, _, err := gojwt.NewParser().ParseUnverified(byJwt, claims); err != nil {