From 879c9afee8b019e94e85e7e209401c34f9854448 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:32:10 +0100 Subject: [PATCH 1/9] fix(firestore-bigquery-export): accept ISO 8601 string partition values Restores 0.2.x behavior for Firestore string partition fields. The 0.3.0 refactor of PartitionValueConverter narrowed accepted inputs to Firestore Timestamp / timestamp-like / Date, silently coercing strings (including ISO 8601 dates such as "2026-01-01") to NULL and landing rows in the __NULL__ partition. PartitionValueConverter.convert now parses strings via new Date(value). Unparseable strings still return null and trigger the existing warning. Adds a defensive try/catch around the BigQuery formatter switch so any serialization failure degrades to null + warn rather than crashing the row write. Fixes #2803 --- firestore-bigquery-export/CHANGELOG.md | 4 ++ .../bigquery/partitioning/converter.test.ts | 63 ++++++++++++++++++- .../src/bigquery/partitioning/converter.ts | 24 ++++--- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 6f6e48a63..38465ee0a 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.3.2 + +fix: restore acceptance of ISO 8601 date/datetime strings as partition field values, regression introduced in 0.3.0 (#2803) + ## Version 0.3.1 chore: bump dependencies diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 19fe6604a..570ea4ab1 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -42,9 +42,22 @@ describe("PartitionValueConverter", () => { expect(result).toBeNull(); }); - test("returns null for string", () => { + test("converts ISO 8601 datetime string to BigQuery timestamp string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toContain("2024-01-15"); + }); + + test("converts ISO 8601 date-only string to BigQuery timestamp string", () => { const result = converter.convert("2024-01-15"); - expect(result).toBeNull(); + expect(result).toContain("2024-01-15"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); }); test("returns null for null", () => { @@ -104,6 +117,32 @@ describe("PartitionValueConverter", () => { const result = converter.convert(date); expect(result).toBe("2024-01-15"); }); + + test("converts ISO 8601 date-only string to BigQuery date string", () => { + const result = converter.convert("2024-01-15"); + expect(result).toBe("2024-01-15"); + }); + + test("converts ISO 8601 datetime string to BigQuery date string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toBe("2024-01-15"); + }); + + test("uses UTC date component for timezone-suffixed datetime string", () => { + // 2024-01-15T22:00:00-08:00 == 2024-01-16T06:00:00Z. The DATE column + // takes the UTC date component, matching how Firestore Timestamps are + // handled. Pinned so future changes to this contract are explicit. + const result = converter.convert("2024-01-15T22:00:00-08:00"); + expect(result).toBe("2024-01-16"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); + }); }); describe("convert with DATETIME type", () => { @@ -134,5 +173,25 @@ describe("PartitionValueConverter", () => { expect(result).toBeDefined(); expect(result).toContain("2024-01-15"); }); + + test("converts ISO 8601 datetime string to BigQuery datetime string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toBeDefined(); + expect(result).toContain("2024-01-15"); + }); + + test("converts ISO 8601 date-only string to BigQuery datetime string", () => { + const result = converter.convert("2024-01-15"); + expect(result).toBeDefined(); + expect(result).toContain("2024-01-15"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); + }); }); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index c1a6a4849..8c7b31f94 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -27,17 +27,27 @@ export class PartitionValueConverter { ).toDate(); } else if (value instanceof Date && !isNaN(value.getTime())) { date = value; + } else if (typeof value === "string") { + const parsed = new Date(value); + if (isNaN(parsed.getTime())) { + return null; + } + date = parsed; } else { return null; } - switch (this.fieldType) { - case "DATETIME": - return BigQuery.datetime(date.toISOString()).value; - case "DATE": - return BigQuery.date(date.toISOString().substring(0, 10)).value; - case "TIMESTAMP": - return BigQuery.timestamp(date).value; + try { + switch (this.fieldType) { + case "DATETIME": + return BigQuery.datetime(date.toISOString()).value; + case "DATE": + return BigQuery.date(date.toISOString().substring(0, 10)).value; + case "TIMESTAMP": + return BigQuery.timestamp(date).value; + } + } catch { + return null; } } } From d0654faf96c90237cb42c8553af70bf7d3b6dcb2 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:41:10 +0100 Subject: [PATCH 2/9] fix(firestore-bigquery-export): explicit default in partition value switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds default: return null to PartitionValueConverter.convert. fieldType is typed as a strict union, so exhaustiveness is already enforced at compile time, but the value comes from external config — a runtime mismatch falls through cleanly to null instead of returning undefined. Per gemini-code-assist review on #2813. --- .../src/bigquery/partitioning/converter.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index 8c7b31f94..aaed055fc 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -45,6 +45,8 @@ export class PartitionValueConverter { return BigQuery.date(date.toISOString().substring(0, 10)).value; case "TIMESTAMP": return BigQuery.timestamp(date).value; + default: + return null; } } catch { return null; From 2d62fdbf88facd9aa65d46d24d642919fa72f96f Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:46:02 +0100 Subject: [PATCH 3/9] chore(firestore-bigquery-export): bump version to 0.3.2 --- firestore-bigquery-export/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 0da2359f0..506e4caf5 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.3.1 +version: 0.3.2 specVersion: v1beta displayName: Stream Firestore to BigQuery From 4392a4f7d60649f6840295081a32adbe893d5d38 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 11:39:43 +0100 Subject: [PATCH 4/9] fix(firestore-bigquery-export): strict ISO 8601 validation for partition strings JavaScript's Date parser is too permissive for partition value validation: new Date("2024-02-30") -> 2024-03-01 (silent month overflow) new Date("2024-01") -> 2024-01-01 (silent partial-date fill) new Date("1") -> 2001-01-01 (bare numeric as year) new Date("2023-02-29") -> 2023-03-01 (non-leap-year overflow) new Date("2024-01-15T10:30") -> engine-dependent local-time interpretation Replaces the loose new Date() check with a strict YYYY-MM-DD prefix regex that requires an explicit timezone designator (Z or +/-HH:MM) when a time component is present, plus a calendar validator built via setUTCFullYear that rejects calendar-invalid components like Feb 30, non-leap-year Feb 29, month 13, and day 32. Per CorieW review on #2813. --- .../bigquery/partitioning/converter.test.ts | 37 +++++++++++++++++++ .../src/bigquery/partitioning/converter.ts | 26 +++++++++++++ 2 files changed, 63 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 570ea4ab1..7d6e7400f 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -60,6 +60,43 @@ describe("PartitionValueConverter", () => { expect(converter.convert("")).toBeNull(); }); + test("returns null for partial date (year-month only)", () => { + expect(converter.convert("2024-01")).toBeNull(); + }); + + test("returns null for partial date (year only)", () => { + expect(converter.convert("2024")).toBeNull(); + }); + + test("returns null for bare numeric string", () => { + expect(converter.convert("1")).toBeNull(); + }); + + test("returns null for calendar-invalid date (Feb 30)", () => { + expect(converter.convert("2024-02-30")).toBeNull(); + }); + + test("returns null for non-leap-year Feb 29", () => { + expect(converter.convert("2023-02-29")).toBeNull(); + }); + + test("accepts leap-year Feb 29", () => { + const result = converter.convert("2024-02-29"); + expect(result).toContain("2024-02-29"); + }); + + test("returns null for out-of-range month", () => { + expect(converter.convert("2024-13-01")).toBeNull(); + }); + + test("returns null for out-of-range day", () => { + expect(converter.convert("2024-01-32")).toBeNull(); + }); + + test("returns null for datetime without timezone", () => { + expect(converter.convert("2024-01-15T10:30:00")).toBeNull(); + }); + test("returns null for null", () => { const result = converter.convert(null); expect(result).toBeNull(); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index aaed055fc..2124c7a0a 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -28,6 +28,32 @@ export class PartitionValueConverter { } else if (value instanceof Date && !isNaN(value.getTime())) { date = value; } else if (typeof value === "string") { + // Strict ISO 8601 / RFC 3339: YYYY-MM-DD, optionally followed by T or + // space-separated HH:MM[:SS[.ffffff]] and a required timezone designator + // when the time component is present. JS Date parsing alone is too + // permissive — it silently normalizes invalid inputs (e.g. "2024-02-30" + // → "2024-03-01"), accepts partial dates ("2024-01"), and reads bare + // numerics as years ("1" → "2001-01-01"). Reject all of those. + const m = value.match( + /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2}))?$/ + ); + if (!m) { + return null; + } + const yearN = Number(m[1]); + const monthN = Number(m[2]); + const dayN = Number(m[3]); + // Reject calendar-invalid components (Feb 30, non-leap Feb 29, etc.). + // setUTCFullYear avoids the legacy 2-digit-year quirk of Date.UTC(). + const validator = new Date(0); + validator.setUTCFullYear(yearN, monthN - 1, dayN); + if ( + validator.getUTCFullYear() !== yearN || + validator.getUTCMonth() + 1 !== monthN || + validator.getUTCDate() !== dayN + ) { + return null; + } const parsed = new Date(value); if (isNaN(parsed.getTime())) { return null; From fba4ca3a263107778921428e6e795c6f92ed682a Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 11:48:36 +0100 Subject: [PATCH 5/9] fix(firestore-bigquery-export): reject year 0 in partition strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BigQuery DATE / DATETIME / TIMESTAMP all reject year 0 — the supported range is 0001-01-01 to 9999-12-31. The previous strict validator passed "0000-01-01" through (setUTCFullYear(0, 0, 1) yields year 0, matching input components) but BigQuery rejects it server-side, causing the same NULL-partition symptom this PR is fixing. Reject yearN < 1 client-side so the user gets a clear warning log. --- .../bigquery/partitioning/converter.test.ts | 14 ++++++++++++++ .../src/bigquery/partitioning/converter.ts | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 7d6e7400f..f1a70b1c1 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -93,6 +93,20 @@ describe("PartitionValueConverter", () => { expect(converter.convert("2024-01-32")).toBeNull(); }); + test("returns null for year 0 (outside BigQuery DATE range)", () => { + expect(converter.convert("0000-01-01")).toBeNull(); + }); + + test("accepts year 0001 (BigQuery DATE minimum)", () => { + const result = converter.convert("0001-01-01"); + expect(result).toContain("0001-01-01"); + }); + + test("accepts year 9999 (BigQuery DATE maximum)", () => { + const result = converter.convert("9999-12-31"); + expect(result).toContain("9999-12-31"); + }); + test("returns null for datetime without timezone", () => { expect(converter.convert("2024-01-15T10:30:00")).toBeNull(); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index 2124c7a0a..2ede51498 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -43,6 +43,12 @@ export class PartitionValueConverter { const yearN = Number(m[1]); const monthN = Number(m[2]); const dayN = Number(m[3]); + // BigQuery DATE / DATETIME / TIMESTAMP all reject year 0 — the supported + // range is 0001-01-01 to 9999-12-31. Reject client-side so the row gets + // a clear warning instead of a server-side insert error. + if (yearN < 1) { + return null; + } // Reject calendar-invalid components (Feb 30, non-leap Feb 29, etc.). // setUTCFullYear avoids the legacy 2-digit-year quirk of Date.UTC(). const validator = new Date(0); From 0f063a6cd92f879720a26d95a8070ce5e1b65263 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 11 May 2026 10:43:57 +0100 Subject: [PATCH 6/9] test(firestore-bigquery-export): cover partition string edge cases Adds 4 cases to PartitionValueConverter TIMESTAMP suite: - out-of-range hour ("25:00:00Z") -> null - out-of-range minute ("23:60:00Z") -> null - timezone offset without colon ("+0800") -> accepted - fractional seconds beyond millisecond precision -> accepted The first two pin the implicit time-validation behavior (regex matches H/M/S as 2-digit pairs, then new Date() rejects out-of-range values). The last two document accepted RFC 3339 variants. --- .../bigquery/partitioning/converter.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index f1a70b1c1..a9704495c 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -111,6 +111,24 @@ describe("PartitionValueConverter", () => { expect(converter.convert("2024-01-15T10:30:00")).toBeNull(); }); + test("returns null for out-of-range hour", () => { + expect(converter.convert("2024-01-15T25:00:00Z")).toBeNull(); + }); + + test("returns null for out-of-range minute", () => { + expect(converter.convert("2024-01-15T23:60:00Z")).toBeNull(); + }); + + test("accepts timezone offset without colon", () => { + const result = converter.convert("2024-01-15T10:30:00+0800"); + expect(result).toContain("2024-01-15"); + }); + + test("accepts fractional seconds beyond millisecond precision", () => { + const result = converter.convert("2024-01-15T10:30:00.123456Z"); + expect(result).toContain("2024-01-15"); + }); + test("returns null for null", () => { const result = converter.convert(null); expect(result).toBeNull(); From e73989bb77170bf8c76cc1f762afb037bf3e8270 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 11 May 2026 11:16:31 +0100 Subject: [PATCH 7/9] test(firestore-bigquery-export): pin DATETIME canonical output form Adds a strict-equality test that asserts the DATETIME converter outputs "2024-01-15 10:30:00.000" (space separator, no Z) when fed an ISO 8601 string with a Z suffix. The existing tests only checked that the result contained "2024-01-15", which left the canonical-form contract unpinned. @google-cloud/bigquery's BigQuery.datetime() already normalises Z-suffixed input to BigQuery's canonical DATETIME form, so the production code is correct. This test guards against future regressions. --- .../__tests__/bigquery/partitioning/converter.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index a9704495c..c44f01c7d 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -255,6 +255,17 @@ describe("PartitionValueConverter", () => { expect(result).toContain("2024-01-15"); }); + test("DATETIME output uses BigQuery canonical form (no Z, space separator)", () => { + // BigQuery DATETIME columns reject the 'Z' timezone suffix and require + // a space (not 'T') between date and time. @google-cloud/bigquery's + // BigQuery.datetime() helper already normalises ISO 8601 input to this + // canonical form, so feeding it date.toISOString() (which always ends + // in 'Z') is safe. Pinned so the contract does not silently regress. + expect(converter.convert("2024-01-15T10:30:00Z")).toBe( + "2024-01-15 10:30:00.000" + ); + }); + test("returns null for unparseable string", () => { expect(converter.convert("not-a-date")).toBeNull(); }); From b0ef05dc85c1258031e1d9407ab5d315288458c9 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 11 May 2026 11:43:02 +0100 Subject: [PATCH 8/9] test(firestore-bigquery-export): cover space-separated partition strings The regex in PartitionValueConverter accepts both "T" and a literal space as the date/time separator (RFC 3339 allows either), but no test covered the space form. Pinned so the contract is explicit. --- .../src/__tests__/bigquery/partitioning/converter.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index c44f01c7d..e3896f23a 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -129,6 +129,11 @@ describe("PartitionValueConverter", () => { expect(result).toContain("2024-01-15"); }); + test("accepts space separator between date and time (RFC 3339 alt form)", () => { + const result = converter.convert("2024-01-15 10:30:00Z"); + expect(result).toContain("2024-01-15"); + }); + test("returns null for null", () => { const result = converter.convert(null); expect(result).toBeNull(); From 6a27f0d43bc95e654cf70a5cda17f1e75f8205e8 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 11 May 2026 12:05:05 +0100 Subject: [PATCH 9/9] fix(firestore-bigquery-export): reject hour 24 in partition strings ISO 8601 allows "24:00:00" as an alias for next-day midnight, and JS Date parses it that way. The new string parsing path in 0.3.2 would therefore silently roll inputs like "2024-01-15T24:00:00Z" forward to the "2024-01-16" partition. That is a regression vs 0.2.x, where the raw string was passed straight to BigQuery and BigQuery rejected hour 24 outright, surfacing the bad row instead of misfiling it. Reject hour 24 explicitly so the caller logs firestoreTimePartitionFieldError and the row lands in the __NULL__ partition (loud failure, matching the calendar-validator philosophy already applied to Feb 30 etc.). Minute and second out-of-range values are still caught by new Date() returning Invalid Date in the existing check below. Adds a test pinning the hour 24 rejection. --- .../__tests__/bigquery/partitioning/converter.test.ts | 9 +++++++++ .../src/bigquery/partitioning/converter.ts | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index e3896f23a..4a77a537a 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -115,6 +115,15 @@ describe("PartitionValueConverter", () => { expect(converter.convert("2024-01-15T25:00:00Z")).toBeNull(); }); + test("returns null for hour 24 (avoids silent next-day shift)", () => { + // ISO 8601 allows 24:00:00 as end-of-day, equivalent to next day 00:00. + // JS Date parses it as such and rolls forward, which would silently + // misfile the row into the next-day partition. 0.2.x passed the raw + // string to BigQuery, which rejected hour=24 outright. Reject here to + // match the loud-failure behavior rather than silent misfiling. + expect(converter.convert("2024-01-15T24:00:00Z")).toBeNull(); + }); + test("returns null for out-of-range minute", () => { expect(converter.convert("2024-01-15T23:60:00Z")).toBeNull(); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index 2ede51498..8d008d635 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -60,6 +60,16 @@ export class PartitionValueConverter { ) { return null; } + // Reject hour 24 (ISO 8601 allows it as end-of-day, but JS Date and the + // pre-0.3.0 string passthrough both treat it differently: JS rolls to + // next day, BigQuery DATETIME rejects the row outright. Rather than + // silently misfile the row into the next-day partition, reject here so + // the caller logs firestoreTimePartitionFieldError and the row lands in + // __NULL__. Minute and second out-of-range values are caught by + // new Date() returning Invalid Date below. + if (m[4] !== undefined && Number(m[4]) > 23) { + return null; + } const parsed = new Date(value); if (isNaN(parsed.getTime())) { return null;