diff --git a/lib/apify/__tests__/persistPostsForSocial.test.ts b/lib/apify/__tests__/persistPostsForSocial.test.ts new file mode 100644 index 00000000..410a49b4 --- /dev/null +++ b/lib/apify/__tests__/persistPostsForSocial.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +import { persistPostsForSocial } from "../persistPostsForSocial"; +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; +import { selectSocials } from "@/lib/supabase/socials/selectSocials"; +import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; + +vi.mock("@/lib/supabase/posts/upsertPosts", () => ({ upsertPosts: vi.fn() })); +vi.mock("@/lib/supabase/posts/getPosts", () => ({ getPosts: vi.fn() })); +vi.mock("@/lib/supabase/socials/selectSocials", () => ({ selectSocials: vi.fn() })); +vi.mock("@/lib/supabase/social_posts/upsertSocialPosts", () => ({ upsertSocialPosts: vi.fn() })); + +const PROFILE_URL = "tiktok.com/@brauxelion"; +const SOCIAL = { id: "soc-1", profile_url: PROFILE_URL }; +const POST_ROWS = [ + { + post_url: "https://www.tiktok.com/@brauxelion/video/1", + updated_at: "2026-06-15T19:18:17.000Z", + }, +]; +const DB_POSTS = [ + { id: "post-1", post_url: POST_ROWS[0].post_url, updated_at: POST_ROWS[0].updated_at }, +]; + +describe("persistPostsForSocial", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getPosts).mockResolvedValue(DB_POSTS as never); + vi.mocked(selectSocials).mockResolvedValue([SOCIAL as never]); + }); + + it("upserts posts and links them to the social row", async () => { + const result = await persistPostsForSocial({ postRows: POST_ROWS, profileUrl: PROFILE_URL }); + expect(upsertPosts).toHaveBeenCalledWith(POST_ROWS); + expect(getPosts).toHaveBeenCalledWith({ postUrls: [POST_ROWS[0].post_url] }); + expect(selectSocials).toHaveBeenCalledWith({ profile_url: PROFILE_URL }); + expect(upsertSocialPosts).toHaveBeenCalledWith([ + { post_id: "post-1", social_id: "soc-1", updated_at: DB_POSTS[0].updated_at }, + ]); + expect(result).toEqual({ posts: DB_POSTS, social: SOCIAL }); + }); + + it("no-ops when there are no post rows", async () => { + const result = await persistPostsForSocial({ postRows: [], profileUrl: PROFILE_URL }); + expect(upsertPosts).not.toHaveBeenCalled(); + expect(upsertSocialPosts).not.toHaveBeenCalled(); + expect(result).toEqual({ posts: [], social: null }); + }); + + it("skips linking when the social row is not found", async () => { + vi.mocked(selectSocials).mockResolvedValue([]); + const result = await persistPostsForSocial({ postRows: POST_ROWS, profileUrl: PROFILE_URL }); + expect(upsertPosts).toHaveBeenCalledWith(POST_ROWS); + expect(upsertSocialPosts).not.toHaveBeenCalled(); + expect(result).toEqual({ posts: DB_POSTS, social: null }); + }); +}); diff --git a/lib/apify/__tests__/toIsoDate.test.ts b/lib/apify/__tests__/toIsoDate.test.ts new file mode 100644 index 00000000..1f57890a --- /dev/null +++ b/lib/apify/__tests__/toIsoDate.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect } from "vitest"; + +import { toIsoDate } from "../toIsoDate"; + +describe("toIsoDate", () => { + it("converts Twitter's legacy date format to ISO", () => { + expect(toIsoDate("Thu Jul 02 17:21:21 +0000 2026")).toBe("2026-07-02T17:21:21.000Z"); + }); + + it("normalizes an already-ISO string", () => { + expect(toIsoDate("2025-06-15T19:18:17.000Z")).toBe("2025-06-15T19:18:17.000Z"); + }); + + it("returns undefined for an unparseable value", () => { + expect(toIsoDate("not-a-date")).toBeUndefined(); + }); + + it("returns undefined for empty/absent values", () => { + expect(toIsoDate("")).toBeUndefined(); + expect(toIsoDate(undefined)).toBeUndefined(); + }); +}); diff --git a/lib/apify/persistPostsForSocial.ts b/lib/apify/persistPostsForSocial.ts new file mode 100644 index 00000000..e9ed1ee7 --- /dev/null +++ b/lib/apify/persistPostsForSocial.ts @@ -0,0 +1,44 @@ +import { upsertPosts } from "@/lib/supabase/posts/upsertPosts"; +import { getPosts } from "@/lib/supabase/posts/getPosts"; +import { selectSocials } from "@/lib/supabase/socials/selectSocials"; +import { upsertSocialPosts } from "@/lib/supabase/social_posts/upsertSocialPosts"; +import type { Tables, TablesInsert } from "@/types/database.types"; + +type PersistPostsForSocialParams = { + postRows: TablesInsert<"posts">[]; + profileUrl: string; +}; + +/** + * Persists scraped post rows and links them to the social row matching + * `profileUrl` — the posts half of what the Instagram profile handler does, + * shared by the TikTok and X/Twitter handlers (recoupable/chat#1840). + * + * Call after `upsertSocials` so the social row exists for first-seen + * profiles. `profileUrl` must be the same normalized key the caller + * upserted, or the lookup misses and posts persist unlinked. + * + * @param params - Post rows plus the normalized social profile_url to link to + * @returns The persisted posts and the linked social row (null when not found) + */ +export async function persistPostsForSocial({ postRows, profileUrl }: PersistPostsForSocialParams) { + if (postRows.length === 0) return { posts: [], social: null }; + + await upsertPosts(postRows); + const posts = await getPosts({ postUrls: postRows.map(p => p.post_url) }); + + const matches = await selectSocials({ profile_url: profileUrl }); + const social: Tables<"socials"> | null = matches?.[0] ?? null; + + if (social && posts.length) { + await upsertSocialPosts( + posts.map(post => ({ + post_id: post.id, + social_id: social.id, + updated_at: post.updated_at, + })), + ); + } + + return { posts, social }; +} diff --git a/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts b/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts index bbef9e4a..26c23068 100644 --- a/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts +++ b/lib/apify/tiktok/__tests__/handleTiktokProfileScraperResults.test.ts @@ -6,14 +6,21 @@ const upsertSocials = vi.fn(); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: (...a: unknown[]) => upsertSocials(...a), })); +const persistPostsForSocial = vi.fn(); +vi.mock("@/lib/apify/persistPostsForSocial", () => ({ + persistPostsForSocial: (...a: unknown[]) => persistPostsForSocial(...a), +})); const payload = { eventData: { actorId: "GdWCkxBtKWOsKjdch" }, resource: { defaultDatasetId: "ds-1" }, } as never; -// Trimmed from real run G4YRI0eUI0d5IidDN (2026-07-01). +// Trimmed from real run G4YRI0eUI0d5IidDN (2026-07-01); post fields +// (webVideoUrl, createTimeISO) verified on real run 9AYX8xyaHWyHtnGtC (2026-07-02). const REAL_ITEM = { text: "In welcher Stadt tanzen wir?", + webVideoUrl: "https://www.tiktok.com/@apache_207/video/7516257593208147205", + createTimeISO: "2025-06-15T19:18:17.000Z", authorMeta: { name: "apache_207", profileUrl: "https://www.tiktok.com/@apache_207", @@ -26,6 +33,7 @@ const REAL_ITEM = { beforeEach(() => { vi.clearAllMocks(); upsertSocials.mockResolvedValue([]); + persistPostsForSocial.mockResolvedValue({ posts: [], social: null }); }); describe("handleTiktokProfileScraperResults", () => { @@ -43,9 +51,41 @@ describe("handleTiktokProfileScraperResults", () => { }, ]); }); + it("persists post rows from the dataset items, linked to the social", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM, { text: "no url item" }] }); + persistPostsForSocial.mockResolvedValue({ posts: [{ id: "p1" }], social: { id: "s1" } }); + await handleTiktokProfileScraperResults(payload); + expect(persistPostsForSocial).toHaveBeenCalledWith({ + postRows: [ + { + post_url: "https://www.tiktok.com/@apache_207/video/7516257593208147205", + updated_at: "2025-06-15T19:18:17.000Z", + }, + ], + profileUrl: "tiktok.com/@apache_207", + }); + }); + it("drops an unparseable createTimeISO instead of forwarding it to the upsert", async () => { + listItems.mockResolvedValue({ + items: [ + { + ...REAL_ITEM, + webVideoUrl: "https://www.tiktok.com/@apache_207/video/1", + createTimeISO: "garbage", + }, + ], + }); + persistPostsForSocial.mockResolvedValue({ posts: [], social: null }); + await handleTiktokProfileScraperResults(payload); + expect(persistPostsForSocial).toHaveBeenCalledWith({ + postRows: [{ post_url: "https://www.tiktok.com/@apache_207/video/1", updated_at: undefined }], + profileUrl: "tiktok.com/@apache_207", + }); + }); it("no-ops when the dataset is empty or has no authorMeta", async () => { listItems.mockResolvedValue({ items: [] }); expect(await handleTiktokProfileScraperResults(payload)).toEqual({ social: null }); expect(upsertSocials).not.toHaveBeenCalled(); + expect(persistPostsForSocial).not.toHaveBeenCalled(); }); }); diff --git a/lib/apify/tiktok/handleTiktokProfileScraperResults.ts b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts index 3ff3ac24..f7361d09 100644 --- a/lib/apify/tiktok/handleTiktokProfileScraperResults.ts +++ b/lib/apify/tiktok/handleTiktokProfileScraperResults.ts @@ -1,10 +1,16 @@ import apifyClient from "@/lib/apify/client"; import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; +import { toIsoDate } from "@/lib/apify/toIsoDate"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; +import type { TablesInsert } from "@/types/database.types"; -/** Post item from clockworks~tiktok-scraper (real shape, run G4YRI0eUI0d5IidDN). */ +/** Post item from clockworks~tiktok-scraper (real shape, run G4YRI0eUI0d5IidDN; + * post fields verified on run 9AYX8xyaHWyHtnGtC). */ type TiktokPostItem = { + webVideoUrl?: string; + createTimeISO?: string; authorMeta?: { name?: string; profileUrl?: string; @@ -16,8 +22,9 @@ type TiktokPostItem = { }; /** - * Persists a TikTok profile scrape back to `socials` (upsert on `profile_url`). - * The actor returns post items; the author's profile stats ride on + * Persists a TikTok profile scrape back to `socials` (upsert on `profile_url`) + * and the returned post items to `posts`/`social_posts` (chat#1840). The + * actor returns post items; the author's profile stats ride on * `authorMeta` of any item. */ export async function handleTiktokProfileScraperResults(parsed: ApifyWebhookPayload) { @@ -34,5 +41,13 @@ export async function handleTiktokProfileScraperResults(parsed: ApifyWebhookPayl followingCount: author.following ?? null, }; await upsertSocials([social]); - return { social }; + + const postRows: TablesInsert<"posts">[] = (items as TiktokPostItem[]).flatMap(item => + item.webVideoUrl + ? [{ post_url: item.webVideoUrl, updated_at: toIsoDate(item.createTimeISO) }] + : [], + ); + const { posts } = await persistPostsForSocial({ postRows, profileUrl: social.profile_url }); + + return { social, posts }; } diff --git a/lib/apify/toIsoDate.ts b/lib/apify/toIsoDate.ts new file mode 100644 index 00000000..81a886f7 --- /dev/null +++ b/lib/apify/toIsoDate.ts @@ -0,0 +1,18 @@ +/** + * Converts an actor-supplied date string to ISO 8601, or undefined when the + * value is absent or unparseable (callers let the posts upsert apply its + * column default rather than forwarding a raw string Postgres may reject). + * + * Handles Twitter's legacy format ("Thu Jul 02 17:21:21 +0000 2026") via + * `new Date()` — this service only runs on Node/V8, where that parse is + * deterministic; the unit test pins the exact format so a runtime change + * would fail loudly. + * + * @param value - Raw date string from an Apify actor item + * @returns ISO 8601 string, or undefined when not parseable + */ +export function toIsoDate(value?: string): string | undefined { + if (!value) return undefined; + const parsed = new Date(value); + return isNaN(parsed.getTime()) ? undefined : parsed.toISOString(); +} diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts index fe936d72..f1c03b78 100644 --- a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -6,14 +6,21 @@ const upsertSocials = vi.fn(); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: (...a: unknown[]) => upsertSocials(...a), })); +const persistPostsForSocial = vi.fn(); +vi.mock("@/lib/apify/persistPostsForSocial", () => ({ + persistPostsForSocial: (...a: unknown[]) => persistPostsForSocial(...a), +})); const payload = { eventData: { actorId: "nfp1fpt5gUlBwPcor" }, resource: { defaultDatasetId: "ds-1" }, } as never; -// Trimmed from real run ALVMZYXkh3WHgeGfT (2026-07-01). +// Trimmed from real run ALVMZYXkh3WHgeGfT (2026-07-01); tweet fields +// (url, createdAt) verified on real run bx3asRqfbNnkKgogG (2026-07-02). const REAL_ITEM = { type: "tweet", + url: "https://x.com/TheASF/status/2072732371472748952", + createdAt: "Thu Jul 02 17:21:21 +0000 2026", author: { userName: "TheASF", url: "https://x.com/TheASF", @@ -27,6 +34,7 @@ const REAL_ITEM = { beforeEach(() => { vi.clearAllMocks(); upsertSocials.mockResolvedValue([]); + persistPostsForSocial.mockResolvedValue({ posts: [], social: null }); }); describe("handleTwitterProfileScraperResults", () => { @@ -45,9 +53,24 @@ describe("handleTwitterProfileScraperResults", () => { }, ]); }); + it("persists tweet rows with createdAt converted to ISO, linked to the social", async () => { + listItems.mockResolvedValue({ items: [REAL_ITEM, { type: "tweet" }] }); + persistPostsForSocial.mockResolvedValue({ posts: [{ id: "p1" }], social: { id: "s1" } }); + await handleTwitterProfileScraperResults(payload); + expect(persistPostsForSocial).toHaveBeenCalledWith({ + postRows: [ + { + post_url: "https://x.com/TheASF/status/2072732371472748952", + updated_at: "2026-07-02T17:21:21.000Z", + }, + ], + profileUrl: "x.com/theasf", + }); + }); it("no-ops on an empty dataset", async () => { listItems.mockResolvedValue({ items: [] }); expect(await handleTwitterProfileScraperResults(payload)).toEqual({ social: null }); expect(upsertSocials).not.toHaveBeenCalled(); + expect(persistPostsForSocial).not.toHaveBeenCalled(); }); }); diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts index 50a51ee1..c583cc9c 100644 --- a/lib/apify/twitter/handleTwitterProfileScraperResults.ts +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -1,10 +1,16 @@ import apifyClient from "@/lib/apify/client"; import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; +import { toIsoDate } from "@/lib/apify/toIsoDate"; import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; +import type { TablesInsert } from "@/types/database.types"; -/** Tweet item from apidojo~twitter-scraper-lite (real shape, run ALVMZYXkh3WHgeGfT). */ +/** Tweet item from apidojo~twitter-scraper-lite (real shape, run ALVMZYXkh3WHgeGfT; + * tweet fields verified on run bx3asRqfbNnkKgogG). */ type TweetItem = { + url?: string; + createdAt?: string; author?: { userName?: string; url?: string; @@ -17,8 +23,9 @@ type TweetItem = { }; /** - * Persists an X/Twitter profile scrape back to `socials`. The actor returns - * tweet items; profile stats ride on `author`. The URL is lowercased — + * Persists an X/Twitter profile scrape back to `socials` and the returned + * tweets to `posts`/`social_posts` (chat#1840). The actor returns tweet + * items; profile stats ride on `author`. The URL is lowercased — * X handles are case-insensitive and the actor echoes display casing * (`x.com/TheASF`) while stored rows are lowercase (`x.com/theasf`); * without this the upsert would create a duplicate row. @@ -38,5 +45,14 @@ export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPay region: author.location || null, }; await upsertSocials([social]); - return { social }; + + // Tweet URLs keep the author's display casing — the path segment is the + // case-sensitive status id's context, and unlike profile keys they are + // stored as-is (posts upsert keys on exact post_url). + const postRows: TablesInsert<"posts">[] = (items as TweetItem[]).flatMap(item => + item.url ? [{ post_url: item.url, updated_at: toIsoDate(item.createdAt) }] : [], + ); + const { posts } = await persistPostsForSocial({ postRows, profileUrl: social.profile_url }); + + return { social, posts }; }