diff --git a/firestore-bigquery-export/scripts/import/__tests__/getRowsFromDocs.test.ts b/firestore-bigquery-export/scripts/import/__tests__/getRowsFromDocs.test.ts index baa6a427e..a470ae54b 100644 --- a/firestore-bigquery-export/scripts/import/__tests__/getRowsFromDocs.test.ts +++ b/firestore-bigquery-export/scripts/import/__tests__/getRowsFromDocs.test.ts @@ -1,4 +1,5 @@ import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker"; +import { FIRESTORE_DEFAULT_DATABASE } from "../src/config"; import { getRowsFromDocs } from "../src/helper"; describe("getRowsFromDocs", () => { @@ -20,6 +21,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "users", queryCollectionGroup: false, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const beforeTimestamp = new Date().toISOString(); @@ -62,6 +64,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "regions/{regionId}/countries/{countryId}/cities", queryCollectionGroup: false, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const result = getRowsFromDocs(mockDocs, mockConfig); @@ -99,6 +102,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "my/{testId}/collection", queryCollectionGroup: false, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const result = getRowsFromDocs(mockDocs, mockConfig); @@ -153,6 +157,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "organizations/{orgId}/users", // Template path queryCollectionGroup: true, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const result = getRowsFromDocs(mockDocs, mockConfig); @@ -217,6 +222,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "my/{coolId}/collection", queryCollectionGroup: true, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const myResult = getRowsFromDocs(mockDocs, myConfig); @@ -233,6 +239,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "my_other/{coolId}/collection", queryCollectionGroup: true, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const myOtherResult = getRowsFromDocs(mockDocs, myOtherConfig); @@ -291,6 +298,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "my_other/{coolId}/collection", queryCollectionGroup: true, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const myOtherResult = getRowsFromDocs(mockDocs, myOtherConfig); @@ -305,6 +313,7 @@ describe("getRowsFromDocs", () => { projectId: "test-project", sourceCollectionPath: "my/{coolId}/collection", queryCollectionGroup: true, + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, } as any; const myResult = getRowsFromDocs(mockDocs, myConfig); @@ -328,4 +337,67 @@ describe("getRowsFromDocs", () => { expect(row.documentName).toContain("/my/"); }); }); + + it("uses non-default Firestore database ID in document_name when specified", () => { + const mockDocs = [ + { + id: "doc1", + ref: { + path: "posts/doc1", + }, + data: () => ({ + title: "Test Post", + }), + }, + ] as any[]; + + // Test with non-default database ID + const mockConfig = { + projectId: "test-project", + sourceCollectionPath: "posts", + queryCollectionGroup: false, + firestoreInstanceId: "alpha", + } as any; + + const result = getRowsFromDocs(mockDocs, mockConfig); + + expect(result).toHaveLength(1); + expect(result[0].documentName).toBe( + "projects/test-project/databases/alpha/documents/posts/doc1" + ); + expect(result[0].documentId).toBe("doc1"); + }); + + it("uses non-default Firestore database ID in collection group queries", () => { + const mockDocs = [ + { + id: "doc1", + ref: { + path: "organizations/org1/posts/doc1", + }, + data: () => ({ + title: "Test Post", + }), + }, + ] as any[]; + + // Test with non-default database ID and collection group query + const mockConfig = { + projectId: "test-project", + sourceCollectionPath: "organizations/{orgId}/posts", + queryCollectionGroup: true, + firestoreInstanceId: "alpha", + } as any; + + const result = getRowsFromDocs(mockDocs, mockConfig); + + expect(result).toHaveLength(1); + expect(result[0].documentName).toBe( + "projects/test-project/databases/alpha/documents/organizations/org1/posts/doc1" + ); + expect(result[0].documentId).toBe("doc1"); + expect(result[0].pathParams).toEqual({ + orgId: "org1", + }); + }); }); diff --git a/firestore-bigquery-export/scripts/import/__tests__/multiThreadCollectionGroupBug.test.ts b/firestore-bigquery-export/scripts/import/__tests__/multiThreadCollectionGroupBug.test.ts index d1068ee58..5104ea4b3 100644 --- a/firestore-bigquery-export/scripts/import/__tests__/multiThreadCollectionGroupBug.test.ts +++ b/firestore-bigquery-export/scripts/import/__tests__/multiThreadCollectionGroupBug.test.ts @@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread"; import * as admin from "firebase-admin"; import { CliConfig } from "../src/types"; import * as workerpool from "workerpool"; +import { FIRESTORE_DEFAULT_DATABASE } from "../src/config"; // Mock Firebase Admin to avoid credential issues jest.mock("firebase-admin", () => { @@ -124,6 +125,7 @@ describe("Multi-threaded Collection Group Bug Reproduction", () => { useEmulator: false, rawChangeLogName: "old_data_raw_changelog", cursorPositionFile: "/tmp/test_cursor_position", + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, }; }); diff --git a/firestore-bigquery-export/scripts/import/__tests__/runMultiThread.test.ts b/firestore-bigquery-export/scripts/import/__tests__/runMultiThread.test.ts index ae681c1e0..f5fba0784 100644 --- a/firestore-bigquery-export/scripts/import/__tests__/runMultiThread.test.ts +++ b/firestore-bigquery-export/scripts/import/__tests__/runMultiThread.test.ts @@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread"; import * as admin from "firebase-admin"; import { CliConfig } from "../src/types"; import * as workerpool from "workerpool"; +import { FIRESTORE_DEFAULT_DATABASE } from "../src/config"; // Initialize Firebase Admin (Ensure credentials are set) if (admin.apps.length === 0) { @@ -86,6 +87,7 @@ describe("runMultiThread Partitioning with Firestore", () => { useEmulator: false, rawChangeLogName: "testTable_raw_changelog", cursorPositionFile: "/tmp/test_cursor_position", + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, }; }); diff --git a/firestore-bigquery-export/scripts/import/__tests__/runMultiThreadMock.test.ts b/firestore-bigquery-export/scripts/import/__tests__/runMultiThreadMock.test.ts index 22fa36c59..02fa9e9fc 100644 --- a/firestore-bigquery-export/scripts/import/__tests__/runMultiThreadMock.test.ts +++ b/firestore-bigquery-export/scripts/import/__tests__/runMultiThreadMock.test.ts @@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread"; import * as workerpool from "workerpool"; import { CliConfig } from "../src/types"; import { cpus } from "os"; +import { FIRESTORE_DEFAULT_DATABASE } from "../src/config"; // Mock helper functions jest.mock("../src/helper", () => ({ @@ -97,6 +98,7 @@ describe("runMultiThread", () => { useEmulator: false, rawChangeLogName: "testTable_raw_changelog", cursorPositionFile: "/tmp/cursor", + firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE, }; }); diff --git a/firestore-bigquery-export/scripts/import/src/config.ts b/firestore-bigquery-export/scripts/import/src/config.ts index 2c956374f..4982c6f7e 100644 --- a/firestore-bigquery-export/scripts/import/src/config.ts +++ b/firestore-bigquery-export/scripts/import/src/config.ts @@ -4,6 +4,8 @@ import inquirer from "inquirer"; import { CliConfig, CliConfigError } from "./types"; +export const FIRESTORE_DEFAULT_DATABASE = "(default)"; + const BIGQUERY_VALID_CHARACTERS = /^[a-zA-Z0-9_]+$/; // regex of ^[^/]+(/[^/]+/[^/]+)*$ export const FIRESTORE_VALID_CHARACTERS = new RegExp("^[^/]+(/[^/]+/[^/]+)*$"); diff --git a/firestore-bigquery-export/scripts/import/src/helper.ts b/firestore-bigquery-export/scripts/import/src/helper.ts index 97717763c..125a45915 100644 --- a/firestore-bigquery-export/scripts/import/src/helper.ts +++ b/firestore-bigquery-export/scripts/import/src/helper.ts @@ -13,8 +13,6 @@ import * as util from "util"; import { resolveWildcardIds } from "./config"; -const FIRESTORE_DEFAULT_DATABASE = "(default)"; - // TODO: do we need this logic? export const initializeDataSink = async ( dataSink: FirestoreBigQueryEventHistoryTracker, @@ -78,7 +76,7 @@ export function getRowsFromDocs( rows.push({ timestamp: new Date().toISOString(), // epoch operation: ChangeType.IMPORT, - documentName: `projects/${config.projectId}/databases/${FIRESTORE_DEFAULT_DATABASE}/documents/${path}`, + documentName: `projects/${config.projectId}/databases/${config.firestoreInstanceId}/documents/${path}`, documentId: doc.id, // TODO: fix this type // @ts-expect-error @@ -96,7 +94,7 @@ export function getRowsFromDocs( return { timestamp: new Date().toISOString(), // epoch operation: ChangeType.IMPORT, - documentName: `projects/${config.projectId}/databases/${FIRESTORE_DEFAULT_DATABASE}/documents/${snapshot.ref.path}`, + documentName: `projects/${config.projectId}/databases/${config.firestoreInstanceId}/documents/${snapshot.ref.path}`, documentId: snapshot.id, pathParams: resolveWildcardIds( config.sourceCollectionPath, diff --git a/storage-resize-images/extension.yaml b/storage-resize-images/extension.yaml index b98fb8895..673201f08 100644 --- a/storage-resize-images/extension.yaml +++ b/storage-resize-images/extension.yaml @@ -426,12 +426,6 @@ events: description: Occurs when the function is settled. Provides no customized data other than the context. - - - type: firebase.extensions.storage-resize-images.v1.onStartResize - description: - Occurs when an image resize operation completes successfully. This event - is only triggered when shouldResize returns true and the resize operation - succeeds. # Lifecycle events disabled - backfill feature commented out # lifecycleEvents: # onInstall: diff --git a/storage-resize-images/functions/src/events.ts b/storage-resize-images/functions/src/events.ts index 6b548d766..f3168b4dc 100644 --- a/storage-resize-images/functions/src/events.ts +++ b/storage-resize-images/functions/src/events.ts @@ -60,19 +60,3 @@ export const recordCompletionEvent = async (data: string | object) => { data, }); }; - -export const recordStartResizeEvent = async ({ - subject, - data, -}: { - subject: string; - data: string | object; -}) => { - if (!eventChannel) return; - - return eventChannel.publish({ - type: getEventType("onStartResize"), - subject, - data, - }); -}; diff --git a/storage-resize-images/functions/src/index.ts b/storage-resize-images/functions/src/index.ts index 6a4a380cb..5cb7034f5 100644 --- a/storage-resize-images/functions/src/index.ts +++ b/storage-resize-images/functions/src/index.ts @@ -59,11 +59,6 @@ const generateResizedImageHandler = async ( return; } - await events.recordStartResizeEvent({ - subject: object.name, - data: { input: object }, - }); - const bucket = admin.storage().bucket(object.bucket); const filePath = object.name; // File path in the bucket. const parsedPath = path.parse(filePath); @@ -154,7 +149,6 @@ const generateResizedImageHandler = async ( export const generateResizedImage = functions.storage .object() .onFinalize(async (object, context) => { - await events.recordStartEvent(object); await generateResizedImageHandler(object); await events.recordCompletionEvent({ context }); });