Merge remote-tracking branch 'origin/main' into pr/2192

Author: nicktrn

.changeset/chilled-weeks-switch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix update command version mismatch detection

.changeset/red-rings-marry.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Provide realtime skipColumns option via untamperable public access tokens

.github/workflows/release-helm.yml

@@ -0,0 +1,127 @@
+name: 🧭 Helm Chart Release
+
+on:
+  push:
+    tags:
+      - 'helm-v*'
+  workflow_dispatch:
+    inputs:
+      chart_version:
+        description: 'Chart version to release'
+        required: true
+        type: string
+
+env:
+  REGISTRY: ghcr.io
+  CHART_NAME: trigger
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Lint Helm Chart
+        run: |
+          helm lint ./hosting/k8s/helm/
+
+      - name: Render templates
+        run: |
+          helm template test-release ./hosting/k8s/helm/ \
+            --values ./hosting/k8s/helm/values.yaml \
+            --output-dir ./helm-output
+
+      - name: Validate manifests
+        uses: docker://ghcr.io/yannh/kubeconform:v0.7.0
+        with:
+          entrypoint: '/kubeconform'
+          args: "-summary -output json ./helm-output"
+
+  release:
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # for gh-release
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract version from tag or input
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.chart_version }}"
+          else
+            VERSION="${{ github.ref_name }}"
+            VERSION="${VERSION#helm-v}"
+          fi
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Releasing version: $VERSION"
+
+      - name: Check Chart.yaml version matches release version
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}')
+          echo "Chart.yaml version: $CHART_VERSION"
+          echo "Release version: $VERSION"
+          if [ "$CHART_VERSION" != "$VERSION" ]; then
+            echo "❌ Chart.yaml version does not match release version!"
+            exit 1
+          fi
+          echo "✅ Chart.yaml version matches release version."
+
+      - name: Package Helm Chart
+        run: |
+          helm package ./hosting/k8s/helm/ --destination /tmp/
+
+      - name: Push Helm Chart to GHCR
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz"
+          
+          # Push to GHCR OCI registry
+          helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts"
+
+      - name: Create GitHub Release
+        id: release
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'push'
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: "Helm Chart ${{ steps.version.outputs.version }}"
+          body: |
+            ### Installation
+            ```bash
+            helm upgrade --install trigger \
+              oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts/${{ env.CHART_NAME }} \
+              --version ${{ steps.version.outputs.version }}
+            ```
+            
+            ### Changes
+            See commit history for detailed changes in this release.
+          files: |
+            /tmp/${{ env.CHART_NAME }}-${{ steps.version.outputs.version }}.tgz
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: true
+          prerelease: true

apps/webapp/app/env.server.ts

@@ -831,6 +831,9 @@ const EnvironmentSchema = z.object({
   RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000),
   RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10),
   RUN_REPLICATION_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
+  RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL: z
+    .enum(["log", "error", "warn", "info", "debug"])
+    .default("info"),
   RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS: z.coerce.number().int().default(10_000),
   RUN_REPLICATION_LEADER_LOCK_RETRY_INTERVAL_MS: z.coerce.number().int().default(500),
   RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT: z.string().default("0"),

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -8,7 +8,11 @@ import { TaskRun } from "@trigger.dev/database";
 import { z } from "zod";
 import { env } from "~/env.server";
 import { EngineServiceValidationError } from "~/runEngine/concerns/errors";
-import { AuthenticatedEnvironment, getOneTimeUseToken } from "~/services/apiAuth.server";
+import {
+  ApiAuthenticationResultSuccess,
+  AuthenticatedEnvironment,
+  getOneTimeUseToken,
+} from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server";
@@ -100,11 +104,7 @@ const { action, loader } = createActionApiRoute(
         return json({ error: "Task not found" }, { status: 404 });
       }
 
-      const $responseHeaders = await responseHeaders(
-        result.run,
-        authentication.environment,
-        triggerClient
-      );
+      const $responseHeaders = await responseHeaders(result.run, authentication, triggerClient);
 
       return json(
         {
@@ -133,19 +133,23 @@ const { action, loader } = createActionApiRoute(
 
 async function responseHeaders(
   run: TaskRun,
-  environment: AuthenticatedEnvironment,
+  authentication: ApiAuthenticationResultSuccess,
   triggerClient?: string | null
 ): Promise<Record<string, string>> {
+  const { environment, realtime } = authentication;
+
   const claimsHeader = JSON.stringify({
     sub: environment.id,
     pub: true,
+    realtime,
   });
 
   if (triggerClient === "browser") {
     const claims = {
       sub: environment.id,
       pub: true,
       scopes: [`read:runs:${run.friendlyId}`],
+      realtime,
     };
 
     const jwt = await internal_generateJWT({

apps/webapp/app/routes/realtime.v1.batches.$batchId.ts

@@ -31,6 +31,7 @@ export const loader = createLoaderApiRoute(
       request.url,
       authentication.environment,
       batchRun.id,
+      authentication.realtime,
       request.headers.get("x-trigger-electric-version") ?? undefined
     );
   }

apps/webapp/app/routes/realtime.v1.runs.$runId.ts

@@ -44,6 +44,7 @@ export const loader = createLoaderApiRoute(
       request.url,
       authentication.environment,
       run.id,
+      authentication.realtime,
       request.headers.get("x-trigger-electric-version") ?? undefined
     );
   }

apps/webapp/app/routes/realtime.v1.runs.ts

@@ -29,6 +29,7 @@ export const loader = createLoaderApiRoute(
       request.url,
       authentication.environment,
       searchParams,
+      authentication.realtime,
       request.headers.get("x-trigger-electric-version") ?? undefined
     );
   }

apps/webapp/app/services/apiAuth.server.ts

@@ -23,6 +23,11 @@ const ClaimsSchema = z.object({
   scopes: z.array(z.string()).optional(),
   // One-time use token
   otu: z.boolean().optional(),
+  realtime: z
+    .object({
+      skipColumns: z.array(z.string()).optional(),
+    })
+    .optional(),
 });
 
 type Optional<T, K extends keyof T> = Prettify<Omit<T, K> & Partial<Pick<T, K>>>;
@@ -43,6 +48,9 @@ export type ApiAuthenticationResultSuccess = {
   environment: AuthenticatedEnvironment;
   scopes?: string[];
   oneTimeUse?: boolean;
+  realtime?: {
+    skipColumns?: string[];
+  };
 };
 
 export type ApiAuthenticationResultFailure = {
@@ -151,6 +159,7 @@ export async function authenticateApiKey(
         environment: validationResults.environment,
         scopes: parsedClaims.success ? parsedClaims.data.scopes : [],
         oneTimeUse: parsedClaims.success ? parsedClaims.data.otu : false,
+        realtime: parsedClaims.success ? parsedClaims.data.realtime : undefined,
       };
     }
   }
@@ -233,6 +242,7 @@ async function authenticateApiKeyWithFailure(
         environment: validationResults.environment,
         scopes: parsedClaims.success ? parsedClaims.data.scopes : [],
         oneTimeUse: parsedClaims.success ? parsedClaims.data.otu : false,
+        realtime: parsedClaims.success ? parsedClaims.data.realtime : undefined,
       };
     }
   }

apps/webapp/app/services/realtimeClient.server.ts

@@ -65,6 +65,10 @@ export type RealtimeRunsParams = {
   createdAt?: string;
 };
 
+export type RealtimeRequestOptions = {
+  skipColumns?: string[];
+};
+
 export class RealtimeClient {
   private redis: RedisClient;
   private expiryTimeInSeconds: number;
@@ -124,15 +128,17 @@ export class RealtimeClient {
     url: URL | string,
     environment: RealtimeEnvironment,
     runId: string,
+    requestOptions?: RealtimeRequestOptions,
     clientVersion?: string
   ) {
-    return this.#streamRunsWhere(url, environment, `id='${runId}'`, clientVersion);
+    return this.#streamRunsWhere(url, environment, `id='${runId}'`, requestOptions, clientVersion);
   }
 
   async streamBatch(
     url: URL | string,
     environment: RealtimeEnvironment,
     batchId: string,
+    requestOptions?: RealtimeRequestOptions,
     clientVersion?: string
   ) {
     const whereClauses: string[] = [
@@ -142,13 +148,14 @@ export class RealtimeClient {
 
     const whereClause = whereClauses.join(" AND ");
 
-    return this.#streamRunsWhere(url, environment, whereClause, clientVersion);
+    return this.#streamRunsWhere(url, environment, whereClause, requestOptions, clientVersion);
   }
 
   async streamRuns(
     url: URL | string,
     environment: RealtimeEnvironment,
     params: RealtimeRunsParams,
+    requestOptions?: RealtimeRequestOptions,
     clientVersion?: string
   ) {
     const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`];
@@ -165,7 +172,13 @@ export class RealtimeClient {
 
     const whereClause = whereClauses.join(" AND ");
 
-    const response = await this.#streamRunsWhere(url, environment, whereClause, clientVersion);
+    const response = await this.#streamRunsWhere(
+      url,
+      environment,
+      whereClause,
+      requestOptions,
+      clientVersion
+    );
 
     if (createdAtFilter) {
       const [setCreatedAtFilterError] = await tryCatch(
@@ -256,12 +269,14 @@ export class RealtimeClient {
     url: URL | string,
     environment: RealtimeEnvironment,
     whereClause: string,
+    requestOptions?: RealtimeRequestOptions,
     clientVersion?: string
   ) {
     const electricUrl = this.#constructRunsElectricUrl(
       url,
       environment,
       whereClause,
+      requestOptions,
       clientVersion
     );
 
@@ -272,6 +287,7 @@ export class RealtimeClient {
     url: URL | string,
     environment: RealtimeEnvironment,
     whereClause: string,
+    requestOptions?: RealtimeRequestOptions,
     clientVersion?: string
   ): URL {
     const $url = new URL(url.toString());
@@ -297,13 +313,10 @@ export class RealtimeClient {
       electricUrl.searchParams.set("handle", electricUrl.searchParams.get("shape_id") ?? "");
     }
 
-    const skipColumnsRaw = $url.searchParams.get("skipColumns");
+    let skipColumns = getSkipColumns($url.searchParams, requestOptions);
 
-    if (skipColumnsRaw) {
-      const skipColumns = skipColumnsRaw
-        .split(",")
-        .map((c) => c.trim())
-        .filter((c) => c !== "" && !RESERVED_COLUMNS.includes(c));
+    if (skipColumns.length > 0) {
+      skipColumns = skipColumns.filter((c) => c !== "" && !RESERVED_COLUMNS.includes(c));
 
       electricUrl.searchParams.set(
         "columns",
@@ -543,3 +556,17 @@ declare module "ioredis" {
     ): Result<number, Context>;
   }
 }
+
+function getSkipColumns(searchParams: URLSearchParams, requestOptions?: RealtimeRequestOptions) {
+  if (requestOptions?.skipColumns) {
+    return requestOptions.skipColumns;
+  }
+
+  const skipColumnsRaw = searchParams.get("skipColumns");
+
+  if (skipColumnsRaw) {
+    return skipColumnsRaw.split(",").map((c) => c.trim());
+  }
+
+  return [];
+}