chore(webapp,run-engine): downgrade boundary log noise to warn

Several boundary log sites and customer-input validation paths were
unconditionally logging at error level for failures the system already
handles gracefully (disconnect, retry-skip, return early). This batch
downgrades them to warn or counts them as metrics — visibility is
preserved in stdout / OTel metrics without surfacing them as alerts.

- apiBuilder.server.ts: logBoundaryError helper — warn for AbortError
  and ServiceValidationError at loader/action boundary catches
- handleSocketIo.server.ts: warn for "Worker authentication failed"
  (system disconnects on auth failure; refs TRI-8863)
- waitpointSystem.ts: skip throw and warn when run was canceled
  while suspended (benign cancel-vs-resume race, no checkpoint to
  resume from)
- runAttemptSystem.ts: warn for failed parse/validate of customer's
  flushedMetadata (system already returns gracefully)
- batch-queue/index.ts: warn for non-retryable batch item failures
  via result.skipRetries (queue size limit exceeded, etc.)
- queryPerformanceMonitor.server.ts: slow queries are observability,
  not errors — warn
- timeoutDeployment.server.ts: deployment-state mismatch is a benign
  timeout-vs-completion race — warn
- platform.v3.server.ts: platform_client.failures_total OTel counter
  with {function, kind} labels replaces logger.error from BillingClient
  call sites; helper recordPlatformFailure(fn, kind)
- waitpointCompletionPacket.server.ts: log inner uploadError before
  throwing the wrapper ServiceValidationError so the underlying error
  context isn't lost

Author: ericallam

apps/webapp/app/runEngine/concerns/waitpointCompletionPacket.server.ts

@@ -2,6 +2,7 @@ import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { env } from "~/env.server";
 import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
+import { logger } from "~/services/logger.server";
 import { ServiceValidationError } from "~/v3/services/common.server";
 
 function packetExtensionForDataType(dataType: string): string {
@@ -53,6 +54,11 @@ export async function processWaitpointCompletionPacket(
   );
 
   if (uploadError) {
+    logger.error("Failed to upload large waitpoint to object store", {
+      error: uploadError,
+      filename,
+      environmentId: environment.id,
+    });
     throw new ServiceValidationError("Failed to upload large waitpoint to object store", 500);
   }
 

apps/webapp/app/services/platform.v3.server.ts

@@ -31,6 +31,7 @@ import { newProjectPath, organizationBillingPath } from "~/utils/pathBuilder";
 import { singleton } from "~/utils/singleton";
 import { RedisCacheStore } from "./unkey/redisCacheStore.server";
 import { $replica } from "~/db.server";
+import { metrics } from "@opentelemetry/api";
 
 function initializeClient() {
   if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
@@ -43,6 +44,22 @@ function initializeClient() {
 }
 
 const client = singleton("billingClient", initializeClient);
+// Failures from @trigger.dev/platform billing client calls. Logged at warn
+// (so they stay in stdout) and counted as a metric — Sentry was the wrong
+// tool for this signal because every task invocation hits these paths.
+const platformClientMeter = metrics.getMeter("trigger.dev/platform-client");
+const platformClientFailuresCounter = platformClientMeter.createCounter(
+  "platform_client.failures_total",
+  {
+    description:
+      "Failures returned or thrown by @trigger.dev/platform billing client calls",
+  }
+);
+
+function recordPlatformFailure(fn: string, kind: "caught" | "no_success") {
+  platformClientFailuresCounter.add(1, { function: fn, kind });
+}
+
 
 function initializePlatformCache() {
   const ctx = new DefaultStatefulContext();
@@ -206,7 +223,7 @@ export async function getCurrentPlan(orgId: string) {
     firstDayOfNextMonth.setUTCHours(0, 0, 0, 0);
 
     if (!result.success) {
-      logger.error("Error getting current plan - no success", { orgId, error: result.error });
+      recordPlatformFailure("getCurrentPlan", "no_success");
       return undefined;
     }
 
@@ -222,7 +239,7 @@ export async function getCurrentPlan(orgId: string) {
 
     return { ...result, usage };
   } catch (e) {
-    logger.error("Error getting current plan - caught error", { orgId, error: e });
+    recordPlatformFailure("getCurrentPlan", "caught");
     return undefined;
   }
 }
@@ -233,13 +250,13 @@ export async function getLimits(orgId: string) {
   try {
     const result = await client.currentPlan(orgId);
     if (!result.success) {
-      logger.error("Error getting limits - no success", { orgId, error: result.error });
+      recordPlatformFailure("getLimits", "no_success");
       return undefined;
     }
 
     return result.v3Subscription?.plan?.limits;
   } catch (e) {
-    logger.error("Error getting limits - caught error", { orgId, error: e });
+    recordPlatformFailure("getLimits", "caught");
     return undefined;
   }
 }
@@ -315,7 +332,7 @@ export async function customerPortalUrl(orgId: string, orgSlug: string) {
       returnUrl: `${env.APP_ORIGIN}${organizationBillingPath({ slug: orgSlug })}`,
     });
   } catch (e) {
-    logger.error("Error getting customer portal Url", { orgId, error: e });
+    recordPlatformFailure("customerPortalUrl", "caught");
     return undefined;
   }
 }
@@ -326,12 +343,12 @@ export async function getPlans() {
   try {
     const result = await client.plans();
     if (!result.success) {
-      logger.error("Error getting plans - no success", { error: result.error });
+      recordPlatformFailure("getPlans", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting plans - caught error", { error: e });
+    recordPlatformFailure("getPlans", "caught");
     return undefined;
   }
 }
@@ -408,12 +425,12 @@ export async function setConcurrencyAddOn(organizationId: string, amount: number
   try {
     const result = await client.setAddOn(organizationId, { type: "concurrency", amount });
     if (!result.success) {
-      logger.error("Error setting concurrency add on - no success", { error: result.error });
+      recordPlatformFailure("setConcurrencyAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting concurrency add on - caught error", { error: e });
+    recordPlatformFailure("setConcurrencyAddOn", "caught");
     return undefined;
   }
 }
@@ -424,12 +441,12 @@ export async function setSeatsAddOn(organizationId: string, amount: number) {
   try {
     const result = await client.setAddOn(organizationId, { type: "seats", amount });
     if (!result.success) {
-      logger.error("Error setting seats add on - no success", { error: result.error });
+      recordPlatformFailure("setSeatsAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting seats add on - caught error", { error: e });
+    recordPlatformFailure("setSeatsAddOn", "caught");
     return undefined;
   }
 }
@@ -440,12 +457,12 @@ export async function setBranchesAddOn(organizationId: string, amount: number) {
   try {
     const result = await client.setAddOn(organizationId, { type: "branches", amount });
     if (!result.success) {
-      logger.error("Error setting branches add on - no success", { error: result.error });
+      recordPlatformFailure("setBranchesAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting branches add on - caught error", { error: e });
+    recordPlatformFailure("setBranchesAddOn", "caught");
     return undefined;
   }
 }
@@ -456,12 +473,12 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat
   try {
     const result = await client.usage(organizationId, { from, to });
     if (!result.success) {
-      logger.error("Error getting usage - no success", { error: result.error });
+      recordPlatformFailure("getUsage", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage - caught error", { error: e });
+    recordPlatformFailure("getUsage", "caught");
     return undefined;
   }
 }
@@ -490,12 +507,12 @@ export async function getUsageSeries(organizationId: string, params: UsageSeries
   try {
     const result = await client.usageSeries(organizationId, params);
     if (!result.success) {
-      logger.error("Error getting usage series - no success", { error: result.error });
+      recordPlatformFailure("getUsageSeries", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage series - caught error", { error: e });
+    recordPlatformFailure("getUsageSeries", "caught");
     return undefined;
   }
 }
@@ -514,12 +531,12 @@ export async function reportInvocationUsage(
       additionalData,
     });
     if (!result.success) {
-      logger.error("Error reporting invocation - no success", { error: result.error });
+      recordPlatformFailure("reportInvocationUsage", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error reporting invocation - caught error", { error: e });
+    recordPlatformFailure("reportInvocationUsage", "caught");
     return undefined;
   }
 }
@@ -550,12 +567,12 @@ export async function getEntitlement(
     try {
       const response = await client.getEntitlement(organizationId);
       if (!response.success) {
-        logger.error("Error getting entitlement - no success", { error: response.error });
+        recordPlatformFailure("getEntitlement", "no_success");
         return undefined;
       }
       return response;
     } catch (e) {
-      logger.error("Error getting entitlement - caught error", { error: e });
+      recordPlatformFailure("getEntitlement", "caught");
       return undefined;
     }
   });
@@ -602,7 +619,7 @@ export async function getBillingAlerts(
   if (!client) return undefined;
   const result = await client.getBillingAlerts(organizationId);
   if (!result.success) {
-    logger.error("Error getting billing alert", { error: result.error, organizationId });
+    recordPlatformFailure("getBillingAlert", "no_success");
     throw new Error("Error getting billing alert");
   }
   return result;
@@ -615,7 +632,7 @@ export async function setBillingAlert(
   if (!client) return undefined;
   const result = await client.updateBillingAlerts(organizationId, alert);
   if (!result.success) {
-    logger.error("Error setting billing alert", { error: result.error, organizationId });
+    recordPlatformFailure("setBillingAlert", "no_success");
     throw new Error("Error setting billing alert");
   }
   return result;
@@ -628,11 +645,7 @@ export async function generateRegistryCredentials(
   if (!client) return undefined;
   const result = await client.generateRegistryCredentials(projectId, region);
   if (!result.success) {
-    logger.error("Error generating registry credentials", {
-      error: result.error,
-      projectId,
-      region,
-    });
+    recordPlatformFailure("generateRegistryCredentials", "no_success");
     throw new Error("Failed to generate registry credentials");
   }
 
@@ -651,13 +664,7 @@ export async function enqueueBuild(
   if (!client) return undefined;
   const result = await client.enqueueBuild(projectId, { deploymentId, artifactKey, options });
   if (!result.success) {
-    logger.error("Error enqueuing build", {
-      error: result.error,
-      projectId,
-      deploymentId,
-      artifactKey,
-      options,
-    });
+    recordPlatformFailure("enqueueBuild", "no_success");
     throw new Error("Failed to enqueue build");
   }
 
@@ -672,12 +679,12 @@ export async function getPrivateLinks(
   const [error, result] = await tryCatch(client.getPrivateLinks(organizationId));
 
   if (error) {
-    logger.error("Error getting private links", { organizationId, error });
+    recordPlatformFailure("getPrivateLinks", "caught");
     return undefined;
   }
 
   if (!result.success) {
-    logger.error("Error getting private links - no success", { organizationId, error: result.error });
+    recordPlatformFailure("getPrivateLinks", "no_success");
     return undefined;
   }
 
@@ -693,12 +700,12 @@ export async function createPrivateLink(
   const [error, result] = await tryCatch(client.createPrivateLink(organizationId, body));
 
   if (error) {
-    logger.error("Error creating private link", { organizationId, error });
+    recordPlatformFailure("createPrivateLink", "caught");
     throw error;
   }
 
   if (!result.success) {
-    logger.error("Error creating private link - no success", { organizationId, error: result.error });
+    recordPlatformFailure("createPrivateLink", "no_success");
     throw new Error(result.error ?? "Failed to create private link");
   }
 
@@ -714,12 +721,12 @@ export async function deletePrivateLink(
   const [error, result] = await tryCatch(client.deletePrivateLink(organizationId, connectionId));
 
   if (error) {
-    logger.error("Error deleting private link", { organizationId, connectionId, error });
+    recordPlatformFailure("deletePrivateLink", "caught");
     throw error;
   }
 
   if (!result.success) {
-    logger.error("Error deleting private link - no success", { organizationId, connectionId, error: result.error });
+    recordPlatformFailure("deletePrivateLink", "no_success");
     throw new Error(result.error ?? "Failed to delete private link");
   }
 }
@@ -732,12 +739,12 @@ export async function getPrivateLinkRegions(
   const [error, result] = await tryCatch(client.getPrivateLinkRegions(organizationId));
 
   if (error) {
-    logger.error("Error getting private link regions", { organizationId, error });
+    recordPlatformFailure("getPrivateLinkRegions", "caught");
     return undefined;
   }
 
   if (!result.success) {
-    logger.error("Error getting private link regions - no success", { organizationId, error: result.error });
+    recordPlatformFailure("getPrivateLinkRegions", "no_success");
     return undefined;
   }