From ef675b861ad8743d46a34d05953b89a5bce083a2 Mon Sep 17 00:00:00 2001 From: jmgasper Date: Wed, 12 Aug 2026 19:53:43 +1000 Subject: [PATCH] PM-5839: Correct payment report calendar dates What was broken Date filters could omit the selected end date or, after making it inclusive, show late July 31 SFDC payments as August 1. Monthly BA fees could also place those payments in the next month. Root cause Finance payment timestamps are stored as UTC values without timezone metadata. Date bounds were interpreted as instants or through the database session timezone, while SFDC output applied the America/New_York conversion in the wrong direction. What was changed Interpret report filters as calendar dates with half-open next-day bounds. Convert SFDC New York calendar bounds to UTC for payment storage, return payment dates with the correct New York offset, and group BA fees by the corrected New York date. Any added/updated tests Updated SQL regression coverage for payment accruals, SFDC payments, both BA fee variants, monthly buckets, and report-directory metadata. PostgreSQL 16 checks cover the July 31 boundary and daylight-saving transitions. --- .../payment/member-payment-accrual.sql | 8 ++- sql/reports/sfdc/ba-fees-monthly.sql | 22 ++++++--- sql/reports/sfdc/ba-fees.sql | 14 +++++- sql/reports/sfdc/payments.sql | 20 ++++++-- .../payment/dto/member-payment-accrual.dto.ts | 6 +-- .../payment/payment-reports.service.spec.ts | 7 +-- src/reports/report-directory.data.spec.ts | 11 +++++ src/reports/report-directory.data.ts | 44 +++++++++++++---- src/reports/sfdc/sfdc-reports.dto.ts | 18 ++++--- src/reports/sfdc/sfdc-reports.service.spec.ts | 49 +++++++++++++++++-- 10 files changed, 157 insertions(+), 42 deletions(-) diff --git a/sql/reports/payment/member-payment-accrual.sql b/sql/reports/payment/member-payment-accrual.sql index f3ec48d..41f2298 100644 --- a/sql/reports/payment/member-payment-accrual.sql +++ b/sql/reports/payment/member-payment-accrual.sql @@ -1,7 +1,7 @@ WITH provided_dates AS ( SELECT - NULLIF($1, '')::timestamptz AS start_date, - NULLIF($2, '')::timestamptz AS end_date + NULLIF($1, '')::date AS start_date, + NULLIF($2, '')::date AS end_date ), params AS ( SELECT @@ -50,9 +50,7 @@ recent_payments AS ( JOIN params pr ON TRUE WHERE w.type = 'PAYMENT' AND p.created_at >= pr.start_date - AND p.created_at < ( - DATE_TRUNC('day', pr.end_date) + INTERVAL '1 day' - ) + AND p.created_at < (pr.end_date + INTERVAL '1 day') ), categorized_payments AS ( SELECT diff --git a/sql/reports/sfdc/ba-fees-monthly.sql b/sql/reports/sfdc/ba-fees-monthly.sql index 7501ac9..245a3d9 100644 --- a/sql/reports/sfdc/ba-fees-monthly.sql +++ b/sql/reports/sfdc/ba-fees-monthly.sql @@ -20,8 +20,18 @@ WITH filtered_payments AS ( LEFT JOIN finance.winnings w ON w.winning_id = p.winnings_id WHERE - ($1::timestamptz IS NULL OR p.created_at >= $1::timestamptz) - AND ($2::timestamptz IS NULL OR p.created_at < (DATE_TRUNC('day', $2::timestamptz) + INTERVAL '1 day')) + ( + $1::date IS NULL + OR p.created_at >= ( + ($1::date::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC' + ) + ) + AND ( + $2::date IS NULL + OR p.created_at < ( + (($2::date + 1)::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC' + ) + ) AND ($3::text[] IS NULL OR p.billing_account = ANY($3::text[])) AND ($4::text[] IS NULL OR p.billing_account != ALL($4::text[])) ), @@ -34,17 +44,17 @@ latest_status AS ( ) SELECT fp.billing_account AS "billingAccountId", - TO_CHAR(DATE_TRUNC('month', fp.created_at AT TIME ZONE 'America/New_York'), 'YYYY-MM') AS "month", + TO_CHAR(DATE_TRUNC('month', fp.created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York'), 'YYYY-MM') AS "month", COALESCE(SUM(fp.challenge_fee), 0) AS "totalFees", COALESCE(SUM(fp.total_amount), 0) AS "totalMemberPayments", COUNT(fp.payment_id) AS "paymentCount", - MIN(fp.created_at AT TIME ZONE 'America/New_York')::date AS "earliestPaymentDate", - MAX(fp.created_at AT TIME ZONE 'America/New_York')::date AS "latestPaymentDate", + MIN(fp.created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York')::date AS "earliestPaymentDate", + MAX(fp.created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York')::date AS "latestPaymentDate", ls.payment_status_desc AS "currentPaymentStatus" FROM filtered_payments fp LEFT JOIN latest_status ls ON ls.billing_account = fp.billing_account GROUP BY fp.billing_account, - DATE_TRUNC('month', fp.created_at AT TIME ZONE 'America/New_York'), + DATE_TRUNC('month', fp.created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York'), ls.payment_status_desc ORDER BY fp.billing_account, "month" DESC; diff --git a/sql/reports/sfdc/ba-fees.sql b/sql/reports/sfdc/ba-fees.sql index 5ad9f28..644ce38 100644 --- a/sql/reports/sfdc/ba-fees.sql +++ b/sql/reports/sfdc/ba-fees.sql @@ -19,8 +19,18 @@ WITH filtered_payments AS ( LEFT JOIN finance.winnings w ON w.winning_id = p.winnings_id WHERE - ($1::timestamptz IS NULL OR p.created_at >= $1::timestamptz) - AND ($2::timestamptz IS NULL OR p.created_at < (DATE_TRUNC('day', $2::timestamptz) + INTERVAL '1 day')) + ( + $1::date IS NULL + OR p.created_at >= ( + ($1::date::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC' + ) + ) + AND ( + $2::date IS NULL + OR p.created_at < ( + (($2::date + 1)::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC' + ) + ) AND ($3::text[] IS NULL OR p.billing_account = ANY($3::text[])) AND ($4::text[] IS NULL OR p.billing_account != ALL($4::text[])) ), diff --git a/sql/reports/sfdc/payments.sql b/sql/reports/sfdc/payments.sql index bf463c9..3f483ff 100644 --- a/sql/reports/sfdc/payments.sql +++ b/sql/reports/sfdc/payments.sql @@ -44,7 +44,13 @@ WITH resolved_payment_references AS ( ) SELECT payment_id as "paymentId", - created_at AT TIME ZONE 'America/New_York' as "paymentDate", + TO_CHAR( + created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York', + 'YYYY-MM-DD"T"HH24:MI:SS.MS' + ) || TO_CHAR( + (created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York') - created_at, + 'HH24:MI' + ) as "paymentDate", billing_account as "billingAccountId", payment_status as "paymentStatus", challenge_fee as "challengeFee", @@ -73,8 +79,16 @@ WHERE WHERE m2.handle = ANY($5::text[]) )) AND ($6::text IS NULL OR challenge_name ILIKE '%' || $6 || '%') - AND created_at >= COALESCE($7::timestamptz, (NOW() AT TIME ZONE 'UTC') - INTERVAL '45 days') - AND ($8::timestamptz IS NULL OR created_at < (DATE_TRUNC('day', $8::timestamptz) + INTERVAL '1 day')) + AND created_at >= COALESCE( + ($7::date::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC', + (NOW() AT TIME ZONE 'UTC') - INTERVAL '45 days' + ) + AND ( + $8::date IS NULL + OR created_at < ( + (($8::date + 1)::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC' + ) + ) AND ($9::numeric IS NULL OR total_amount >= $9::numeric) AND ($10::numeric IS NULL OR total_amount <= $10::numeric) AND ($11::text[] IS NULL OR reported_challenge_status::text = ANY($11::text[])) diff --git a/src/reports/payment/dto/member-payment-accrual.dto.ts b/src/reports/payment/dto/member-payment-accrual.dto.ts index e96736a..897e3b3 100644 --- a/src/reports/payment/dto/member-payment-accrual.dto.ts +++ b/src/reports/payment/dto/member-payment-accrual.dto.ts @@ -4,8 +4,8 @@ import { IsDateString, IsOptional } from "class-validator"; export class MemberPaymentAccrualQueryDto { @ApiPropertyOptional({ description: - "Start date (inclusive) for filtering payment creation date in ISO 8601 format", - example: "2024-01-01T00:00:00.000Z", + "Start date (inclusive) for filtering payment creation date in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used", + example: "2024-01-01", }) @IsOptional() @IsDateString() @@ -13,7 +13,7 @@ export class MemberPaymentAccrualQueryDto { @ApiPropertyOptional({ description: - "End date (inclusive through the full calendar day) for filtering payment creation date in ISO 8601 format", + "End date (inclusive through the full calendar day) for filtering payment creation date in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used", example: "2024-01-31", }) @IsOptional() diff --git a/src/reports/payment/payment-reports.service.spec.ts b/src/reports/payment/payment-reports.service.spec.ts index b5a2e16..dfe4f03 100644 --- a/src/reports/payment/payment-reports.service.spec.ts +++ b/src/reports/payment/payment-reports.service.spec.ts @@ -78,10 +78,11 @@ describe("PaymentReportsService", () => { "reports/payment/member-payment-accrual.sql", ); + expect(paymentSql).toContain("NULLIF($1, '')::date AS start_date"); + expect(paymentSql).toContain("NULLIF($2, '')::date AS end_date"); expect(paymentSql).toContain("p.created_at >= pr.start_date"); - expect(paymentSql).toContain( - "DATE_TRUNC('day', pr.end_date) + INTERVAL '1 day'", - ); + expect(paymentSql).toContain("pr.end_date + INTERVAL '1 day'"); expect(paymentSql).not.toContain("p.created_at <= pr.end_date"); + expect(paymentSql).not.toContain("DATE_TRUNC('day', pr.end_date)"); }); }); diff --git a/src/reports/report-directory.data.spec.ts b/src/reports/report-directory.data.spec.ts index 3c75ab3..f233bd3 100644 --- a/src/reports/report-directory.data.spec.ts +++ b/src/reports/report-directory.data.spec.ts @@ -19,6 +19,17 @@ describe("getAccessibleReportsDirectory", () => { "/payment/member-payment-accrual-task", "/payment/member-payment-accrual-challenge", ]); + expect( + directory.sfdc?.reports + .find((report) => report.path === "/sfdc/payments") + ?.parameters?.find((parameter) => parameter.name === "endDate") + ?.description, + ).toContain("America/New_York"); + expect( + directory.payment?.reports[0].parameters?.find( + (parameter) => parameter.name === "endDate", + )?.description, + ).toContain("Inclusive full calendar end date"); expect(directory.statistics?.reports.map((report) => report.path)).toEqual( expect.arrayContaining([ "/statistics/general/country-member-details", diff --git a/src/reports/report-directory.data.ts b/src/reports/report-directory.data.ts index b39b712..8a095e3 100644 --- a/src/reports/report-directory.data.ts +++ b/src/reports/report-directory.data.ts @@ -231,6 +231,28 @@ const paymentsEndDateParam: ReportParameter = { location: "query", }; +const paymentAccrualStartDateParam: ReportParameter = { + ...paymentsStartDateParam, + description: "Inclusive calendar start date in YYYY-MM-DD format", +}; + +const paymentAccrualEndDateParam: ReportParameter = { + ...paymentsEndDateParam, + description: "Inclusive full calendar end date in YYYY-MM-DD format", +}; + +const sfdcPaymentsStartDateParam: ReportParameter = { + ...paymentsStartDateParam, + description: + "Inclusive America/New_York calendar start date in YYYY-MM-DD format", +}; + +const sfdcPaymentsEndDateParam: ReportParameter = { + ...paymentsEndDateParam, + description: + "Inclusive full America/New_York calendar end date in YYYY-MM-DD format", +}; + const challengeNameParam: ReportParameter = { name: "challengeName", type: "string", @@ -294,8 +316,8 @@ const paymentsFilters = [ challengeNameParam, challengeIdsParam, engagementIdsParam, - paymentsStartDateParam, - paymentsEndDateParam, + sfdcPaymentsStartDateParam, + sfdcPaymentsEndDateParam, handlesParam, minPaymentParam, maxPaymentParam, @@ -306,14 +328,16 @@ const baFeesDateParams: ReportParameter[] = [ { name: "startDate", type: "date", - description: "Start date for the report query in ISO format", + description: + "Inclusive America/New_York calendar start date in YYYY-MM-DD format", location: "query", required: true, }, { name: "endDate", type: "date", - description: "End date for the report query in ISO format", + description: + "Inclusive full America/New_York calendar end date in YYYY-MM-DD format", location: "query", }, ]; @@ -837,37 +861,37 @@ const REGISTERED_REPORTS_DIRECTORY: RegisteredReportsDirectory = { "Member Payment Accrual", "/payment/member-payment-accrual", "Member payment accruals for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), adminOnlyTopcoderReport( "Member Payment Accrual-TaaS", "/payment/member-payment-accrual-taas", "Member payment accruals for TaaS payments for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), adminOnlyTopcoderReport( "Member Payment Accrual-Topgear", "/payment/member-payment-accrual-topgear", "Member payment accruals for Topgear payments for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), adminOnlyTopcoderReport( "Member Payment Accrual-Engagement", "/payment/member-payment-accrual-engagement", "Member payment accruals for engagement payments for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), adminOnlyTopcoderReport( "Member Payment Accrual-Task", "/payment/member-payment-accrual-task", "Member payment accruals for task payments for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), adminOnlyTopcoderReport( "Member Payment Accrual-Challenge", "/payment/member-payment-accrual-challenge", "Member payment accruals for challenge payments (contest, review board, copilot, checkpoint, and related challenge payouts) for the provided date range (defaults to last 3 months)", - [paymentsStartDateParam, paymentsEndDateParam], + [paymentAccrualStartDateParam, paymentAccrualEndDateParam], ), ], }, diff --git a/src/reports/sfdc/sfdc-reports.dto.ts b/src/reports/sfdc/sfdc-reports.dto.ts index 20a8ae8..7f3d4db 100644 --- a/src/reports/sfdc/sfdc-reports.dto.ts +++ b/src/reports/sfdc/sfdc-reports.dto.ts @@ -241,8 +241,9 @@ export class PaymentsReportQueryDto { @ApiProperty({ required: false, - description: "Start date for the report query in ISO 8601 format", - example: "2023-01-01T00:00:00.000Z", + description: + "Start date (inclusive from the start of the America/New_York calendar day) for the report query in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used", + example: "2023-01-01", }) @IsOptional() @IsDateString() @@ -251,7 +252,7 @@ export class PaymentsReportQueryDto { @ApiProperty({ required: false, description: - "End date (inclusive through the full calendar day) for the report query in ISO 8601 format", + "End date (inclusive through the full America/New_York calendar day) for the report query in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used", example: "2023-01-31", }) @IsOptional() @@ -332,6 +333,11 @@ export class PaymentsReportResponse { description: "Winnings category from finance.winnings.category", }) category: string; + @ApiProperty({ + description: + "Payment creation timestamp in America/New_York with its UTC offset", + example: "2026-07-31T18:53:33.383-04:00", + }) paymentDate: string; paymentId: string; paymentStatus: string; @@ -854,8 +860,8 @@ export class BaFeesReportQueryDto { @ApiProperty({ required: false, description: - "Start date for the report query in ISO 8601 format (inclusive). If omitted the report uses an open-ended lower bound.", - example: "2023-01-01T00:00:00.000Z", + "Start date (inclusive from the start of the America/New_York calendar day) for the report query in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used. If omitted the report uses an open-ended lower bound.", + example: "2023-01-01", }) @IsOptional() @IsDateString() @@ -864,7 +870,7 @@ export class BaFeesReportQueryDto { @ApiProperty({ required: false, description: - "End date (inclusive through the full calendar day) for the report query in ISO 8601 format", + "End date (inclusive through the full America/New_York calendar day) for the report query in YYYY-MM-DD format. For accepted ISO timestamps, only the written calendar-date portion is used", example: "2023-01-31", }) @IsOptional() diff --git a/src/reports/sfdc/sfdc-reports.service.spec.ts b/src/reports/sfdc/sfdc-reports.service.spec.ts index f9d41ca..2b3fccb 100644 --- a/src/reports/sfdc/sfdc-reports.service.spec.ts +++ b/src/reports/sfdc/sfdc-reports.service.spec.ts @@ -331,16 +331,35 @@ describe("SfdcReportsService - getPaymentsReport", () => { ); }); - it("uses the full inclusive end date in the payments SQL", () => { + it("uses inclusive New York calendar date bounds in the payments SQL", () => { const paymentsSql = readFileSync( join(__dirname, "../../../sql/reports/sfdc/payments.sql"), "utf8", ); expect(paymentsSql).toContain( - "DATE_TRUNC('day', $8::timestamptz) + INTERVAL '1 day'", + "($7::date::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC'", + ); + expect(paymentsSql).toContain( + "(($8::date + 1)::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC'", ); expect(paymentsSql).not.toContain("created_at <= $8::timestamptz"); + expect(paymentsSql).not.toContain("DATE_TRUNC('day', $8::timestamptz)"); + }); + + it("returns offset-bearing New York payment timestamps", () => { + const paymentsSql = readFileSync( + join(__dirname, "../../../sql/reports/sfdc/payments.sql"), + "utf8", + ); + + expect(paymentsSql).toContain( + "created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York',", + ); + expect(paymentsSql).toContain( + "(created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York') - created_at", + ); + expect(paymentsSql).toContain(`'YYYY-MM-DD"T"HH24:MI:SS.MS'`); }); it("returns mixed challenge, engagement, and unresolved challenge payments successfully", async () => { @@ -1121,7 +1140,7 @@ describe("SfdcReportsService - getBaFeesReport", () => { }); it.each(["ba-fees.sql", "ba-fees-monthly.sql"])( - "uses the full inclusive end date in %s", + "uses inclusive New York calendar date bounds in %s", (fileName) => { const baFeesSql = readFileSync( join(__dirname, `../../../sql/reports/sfdc/${fileName}`), @@ -1129,12 +1148,34 @@ describe("SfdcReportsService - getBaFeesReport", () => { ); expect(baFeesSql).toContain( - "DATE_TRUNC('day', $2::timestamptz) + INTERVAL '1 day'", + "($1::date::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC'", + ); + expect(baFeesSql).toContain( + "(($2::date + 1)::timestamp AT TIME ZONE 'America/New_York') AT TIME ZONE 'UTC'", ); expect(baFeesSql).not.toContain("p.created_at <= $2::timestamptz"); + expect(baFeesSql).not.toContain("DATE_TRUNC('day', $2::timestamptz)"); }, ); + it("uses New York calendar dates for monthly BA fee buckets", () => { + const baFeesSql = readFileSync( + join(__dirname, "../../../sql/reports/sfdc/ba-fees-monthly.sql"), + "utf8", + ); + const newYorkPaymentTimestamp = + "fp.created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/New_York'"; + + expect(baFeesSql).toContain( + `DATE_TRUNC('month', ${newYorkPaymentTimestamp})`, + ); + expect(baFeesSql).toContain(`MIN(${newYorkPaymentTimestamp})::date`); + expect(baFeesSql).toContain(`MAX(${newYorkPaymentTimestamp})::date`); + expect(baFeesSql).not.toContain( + "fp.created_at AT TIME ZONE 'America/New_York'", + ); + }); + it("runs a basic query successfully", async () => { const result = await service.getBaFeesReport( mockBaFeesQueryDto.byBillingAccount,