diff --git a/lib/apify/__tests__/getApifyResultHandler.test.ts b/lib/apify/__tests__/getApifyResultHandler.test.ts index 8cfaeab7..9d56d44a 100644 --- a/lib/apify/__tests__/getApifyResultHandler.test.ts +++ b/lib/apify/__tests__/getApifyResultHandler.test.ts @@ -5,10 +5,14 @@ import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInst import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; import { handleTiktokProfileScraperResults } from "@/lib/apify/tiktok/handleTiktokProfileScraperResults"; import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; +import { handleTwitterUserScraperResults } from "@/lib/apify/twitter/handleTwitterUserScraperResults"; import { handleYoutubeProfileScraperResults } from "@/lib/apify/youtube/handleYoutubeProfileScraperResults"; import { handleThreadsProfileScraperResults } from "@/lib/apify/threads/handleThreadsProfileScraperResults"; import { handleFacebookProfileScraperResults } from "@/lib/apify/facebook/handleFacebookProfileScraperResults"; +vi.mock("@/lib/apify/twitter/handleTwitterUserScraperResults", () => ({ + handleTwitterUserScraperResults: vi.fn(), +})); vi.mock("@/lib/apify/instagram/handleInstagramProfileScraperResults", () => ({ handleInstagramProfileScraperResults: vi.fn(), })); @@ -45,6 +49,7 @@ describe("getApifyResultHandler", () => { it("maps every platform's resolved actor id to its results handler", () => { expect(getApifyResultHandler("GdWCkxBtKWOsKjdch")).toBe(handleTiktokProfileScraperResults); expect(getApifyResultHandler("nfp1fpt5gUlBwPcor")).toBe(handleTwitterProfileScraperResults); + expect(getApifyResultHandler("V38PZzpEgOfeeWvZY")).toBe(handleTwitterUserScraperResults); expect(getApifyResultHandler("h7sDV53CddomktSi5")).toBe(handleYoutubeProfileScraperResults); expect(getApifyResultHandler("kJdK90pa2hhYYrCK5")).toBe(handleThreadsProfileScraperResults); expect(getApifyResultHandler("4Hv5RhChiaDk6iwad")).toBe(handleFacebookProfileScraperResults); diff --git a/lib/apify/getApifyResultHandler.ts b/lib/apify/getApifyResultHandler.ts index e2f908ea..22c4d6d5 100644 --- a/lib/apify/getApifyResultHandler.ts +++ b/lib/apify/getApifyResultHandler.ts @@ -3,6 +3,7 @@ import { handleInstagramCommentsScraper } from "@/lib/apify/instagram/handleInst import { handleLinkedinProfileScraperResults } from "@/lib/apify/linkedin/handleLinkedinProfileScraperResults"; import { handleTiktokProfileScraperResults } from "@/lib/apify/tiktok/handleTiktokProfileScraperResults"; import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; +import { handleTwitterUserScraperResults } from "@/lib/apify/twitter/handleTwitterUserScraperResults"; import { handleYoutubeProfileScraperResults } from "@/lib/apify/youtube/handleYoutubeProfileScraperResults"; import { handleThreadsProfileScraperResults } from "@/lib/apify/threads/handleThreadsProfileScraperResults"; import { handleFacebookProfileScraperResults } from "@/lib/apify/facebook/handleFacebookProfileScraperResults"; @@ -26,7 +27,8 @@ const HANDLERS_BY_ACTOR_ID: Record = { SbK00X0JYCPblD2wp: handleInstagramCommentsScraper, // instagram comments LpVuK3Zozwuipa5bp: handleLinkedinProfileScraperResults, // linkedin profile (harvestapi) GdWCkxBtKWOsKjdch: handleTiktokProfileScraperResults, // tiktok (clockworks~tiktok-scraper) - nfp1fpt5gUlBwPcor: handleTwitterProfileScraperResults, // x/twitter (apidojo~twitter-scraper-lite) + nfp1fpt5gUlBwPcor: handleTwitterProfileScraperResults, // x/twitter tweets (apidojo~twitter-scraper-lite) + V38PZzpEgOfeeWvZY: handleTwitterUserScraperResults, // x/twitter profile fallback (apidojo~twitter-user-scraper) h7sDV53CddomktSi5: handleYoutubeProfileScraperResults, // youtube (streamers~youtube-scraper) kJdK90pa2hhYYrCK5: handleThreadsProfileScraperResults, // threads (apify~threads-profile-api-scraper) "4Hv5RhChiaDk6iwad": handleFacebookProfileScraperResults, // facebook (apify~facebook-pages-scraper) diff --git a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts index f1c03b78..88cbecf8 100644 --- a/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts +++ b/lib/apify/twitter/__tests__/handleTwitterProfileScraperResults.test.ts @@ -1,7 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { handleTwitterProfileScraperResults } from "@/lib/apify/twitter/handleTwitterProfileScraperResults"; const listItems = vi.fn(); -vi.mock("@/lib/apify/client", () => ({ default: { dataset: vi.fn(() => ({ listItems })) } })); +const getRecord = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ + default: { dataset: vi.fn(() => ({ listItems })), keyValueStore: vi.fn(() => ({ getRecord })) }, +})); +const startTwitterUserScraping = vi.fn(); +vi.mock("@/lib/apify/twitter/startTwitterUserScraping", () => ({ + default: (...a: unknown[]) => startTwitterUserScraping(...a), +})); const upsertSocials = vi.fn(); vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ upsertSocials: (...a: unknown[]) => upsertSocials(...a), @@ -13,7 +20,7 @@ vi.mock("@/lib/apify/persistPostsForSocial", () => ({ const payload = { eventData: { actorId: "nfp1fpt5gUlBwPcor" }, - resource: { defaultDatasetId: "ds-1" }, + resource: { defaultDatasetId: "ds-1", defaultKeyValueStoreId: "kv-1" }, } as never; // Trimmed from real run ALVMZYXkh3WHgeGfT (2026-07-01); tweet fields // (url, createdAt) verified on real run bx3asRqfbNnkKgogG (2026-07-02). @@ -67,10 +74,33 @@ describe("handleTwitterProfileScraperResults", () => { profileUrl: "x.com/theasf", }); }); - it("no-ops on an empty dataset", async () => { + it("falls back to a profile-level user scrape on an empty dataset (tweetless account, chat#1851)", async () => { listItems.mockResolvedValue({ items: [] }); - expect(await handleTwitterProfileScraperResults(payload)).toEqual({ social: null }); + getRecord.mockResolvedValue({ key: "INPUT", value: { twitterHandles: ["KETTAMA_"] } }); + startTwitterUserScraping.mockResolvedValue({ runId: "run-u1", datasetId: "ds-u1" }); + expect(await handleTwitterProfileScraperResults(payload)).toEqual({ + social: null, + fallbackRunId: "run-u1", + }); + expect(startTwitterUserScraping).toHaveBeenCalledWith("KETTAMA_"); expect(upsertSocials).not.toHaveBeenCalled(); expect(persistPostsForSocial).not.toHaveBeenCalled(); }); + + it("no-ops on an empty dataset when the run INPUT carries no handle", async () => { + listItems.mockResolvedValue({ items: [] }); + getRecord.mockResolvedValue({ key: "INPUT", value: {} }); + expect(await handleTwitterProfileScraperResults(payload)).toEqual({ social: null }); + expect(startTwitterUserScraping).not.toHaveBeenCalled(); + }); + + it("no-ops on an empty dataset when the payload carries no key-value store id", async () => { + listItems.mockResolvedValue({ items: [] }); + const bare = { + eventData: { actorId: "nfp1fpt5gUlBwPcor" }, + resource: { defaultDatasetId: "ds-1" }, + } as never; + expect(await handleTwitterProfileScraperResults(bare)).toEqual({ social: null }); + expect(startTwitterUserScraping).not.toHaveBeenCalled(); + }); }); diff --git a/lib/apify/twitter/__tests__/handleTwitterUserScraperResults.test.ts b/lib/apify/twitter/__tests__/handleTwitterUserScraperResults.test.ts new file mode 100644 index 00000000..488a09db --- /dev/null +++ b/lib/apify/twitter/__tests__/handleTwitterUserScraperResults.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { handleTwitterUserScraperResults } from "@/lib/apify/twitter/handleTwitterUserScraperResults"; + +const listItems = vi.fn(); +const getRecord = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ + default: { + dataset: vi.fn(() => ({ listItems })), + keyValueStore: vi.fn(() => ({ getRecord })), + }, +})); +const upsertSocials = vi.fn(); +vi.mock("@/lib/supabase/socials/upsertSocials", () => ({ + upsertSocials: (...a: unknown[]) => upsertSocials(...a), +})); + +const payload = { + eventData: { actorId: "V38PZzpEgOfeeWvZY" }, + resource: { defaultDatasetId: "ds-u1", defaultKeyValueStoreId: "kv-u1" }, +} as never; + +// Real shape from a live apidojo~twitter-user-scraper run (2026-07-06, +// twitterHandles: ["ashnikko"]): the actor returns the requested user AND +// pads with related users, so the handler must filter by the requested handle. +const REQUESTED_USER = { + type: "user", + userName: "ashnikko", + url: "https://x.com/ashnikko", + profilePicture: "https://pbs.twimg.com/profile_images/ash.jpg", + description: "smoochies out now", + location: "London", + followers: 239289, + following: 500, +}; +const RELATED_PADDING_USER = { + type: "user", + userName: "infoashnikko", + url: "https://x.com/infoashnikko", + followers: 3053, + following: 12, +}; + +beforeEach(() => { + vi.clearAllMocks(); + upsertSocials.mockResolvedValue([]); + getRecord.mockResolvedValue({ + key: "INPUT", + value: { twitterHandles: ["Ashnikko"], maxItems: 1 }, + }); +}); + +describe("handleTwitterUserScraperResults", () => { + it("upserts only the requested handle's profile with a LOWERCASED x.com profile_url", async () => { + listItems.mockResolvedValue({ items: [RELATED_PADDING_USER, REQUESTED_USER] }); + const result = await handleTwitterUserScraperResults(payload); + expect(upsertSocials).toHaveBeenCalledWith([ + { + profile_url: "x.com/ashnikko", + username: "ashnikko", + avatar: "https://pbs.twimg.com/profile_images/ash.jpg", + bio: "smoochies out now", + followerCount: 239289, + followingCount: 500, + region: "London", + }, + ]); + expect(result).toEqual({ social: expect.objectContaining({ profile_url: "x.com/ashnikko" }) }); + }); + + it("returns social null and does not upsert when the requested handle is absent from items", async () => { + listItems.mockResolvedValue({ items: [RELATED_PADDING_USER] }); + const result = await handleTwitterUserScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); + + it("returns social null when the INPUT record is unavailable", async () => { + getRecord.mockResolvedValue(undefined); + listItems.mockResolvedValue({ items: [REQUESTED_USER] }); + const result = await handleTwitterUserScraperResults(payload); + expect(upsertSocials).not.toHaveBeenCalled(); + expect(result).toEqual({ social: null }); + }); +}); diff --git a/lib/apify/twitter/__tests__/startTwitterUserScraping.test.ts b/lib/apify/twitter/__tests__/startTwitterUserScraping.test.ts new file mode 100644 index 00000000..d8cfe5ee --- /dev/null +++ b/lib/apify/twitter/__tests__/startTwitterUserScraping.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import apifyClient from "@/lib/apify/client"; +import startTwitterUserScraping from "@/lib/apify/twitter/startTwitterUserScraping"; + +const start = vi.fn(); +vi.mock("@/lib/apify/client", () => ({ default: { actor: vi.fn(() => ({ start })) } })); + +beforeEach(() => { + vi.clearAllMocks(); + start.mockResolvedValue({ id: "run-u1", defaultDatasetId: "ds-u1", status: "RUNNING" }); +}); + +describe("startTwitterUserScraping", () => { + it("starts the profile-level user scraper for one handle", async () => { + const r = await startTwitterUserScraping("KETTAMA_"); + expect(apifyClient.actor).toHaveBeenCalledWith("apidojo/twitter-user-scraper"); + expect(start).toHaveBeenCalledWith( + { twitterHandles: ["KETTAMA_"], maxItems: 1 }, + { webhooks: expect.any(Array) }, + ); + expect(r).toEqual({ runId: "run-u1", datasetId: "ds-u1" }); + }); + + it("throws on an empty handle", async () => { + await expect(startTwitterUserScraping(" ")).rejects.toThrow(/Invalid Twitter handle/); + expect(start).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/apify/twitter/handleTwitterProfileScraperResults.ts b/lib/apify/twitter/handleTwitterProfileScraperResults.ts index c583cc9c..243d2fd3 100644 --- a/lib/apify/twitter/handleTwitterProfileScraperResults.ts +++ b/lib/apify/twitter/handleTwitterProfileScraperResults.ts @@ -1,4 +1,5 @@ import apifyClient from "@/lib/apify/client"; +import startTwitterUserScraping from "@/lib/apify/twitter/startTwitterUserScraping"; import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; import { persistPostsForSocial } from "@/lib/apify/persistPostsForSocial"; @@ -32,6 +33,22 @@ type TweetItem = { */ export async function handleTwitterProfileScraperResults(parsed: ApifyWebhookPayload) { const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + + // Tweetless accounts return zero items, so profile stats never arrive via + // `author`. Recover the requested handle from the run INPUT and fall back + // to a profile-level user scrape (chat#1851); its results are persisted by + // handleTwitterUserScraperResults through the same webhook receiver. + if (items.length === 0) { + const storeId = parsed.resource.defaultKeyValueStoreId; + if (!storeId) return { social: null }; + const inputRecord = await apifyClient.keyValueStore(storeId).getRecord("INPUT"); + const input = inputRecord?.value as { twitterHandles?: string[] } | undefined; + const handle = input?.twitterHandles?.[0]?.trim(); + if (!handle) return { social: null }; + const fallbackRun = await startTwitterUserScraping(handle); + return { social: null, fallbackRunId: fallbackRun?.runId ?? null }; + } + const author = (items[0] as TweetItem | undefined)?.author; if (!author?.url) return { social: null }; diff --git a/lib/apify/twitter/handleTwitterUserScraperResults.ts b/lib/apify/twitter/handleTwitterUserScraperResults.ts new file mode 100644 index 00000000..04e58c5c --- /dev/null +++ b/lib/apify/twitter/handleTwitterUserScraperResults.ts @@ -0,0 +1,55 @@ +import apifyClient from "@/lib/apify/client"; +import { upsertSocials } from "@/lib/supabase/socials/upsertSocials"; +import { normalizeProfileUrl } from "@/lib/socials/normalizeProfileUrl"; +import type { ApifyWebhookPayload } from "@/lib/apify/validateApifyWebhookRequest"; + +/** User item from apidojo~twitter-user-scraper (real shape verified on a live + * run 2026-07-06, twitterHandles: ["ashnikko"]). The actor returns the + * requested user AND pads the dataset with related users, so results must be + * filtered to the requested handle before persisting. */ +type TwitterUserItem = { + type?: string; + userName?: string; + url?: string; + profilePicture?: string; + description?: string; + location?: string; + followers?: number; + following?: number; +}; + +/** + * Persists a profile-level X/Twitter scrape (the tweetless-account fallback, + * chat#1851) back to `socials`. Reads the run INPUT to recover the requested + * handle, filters the dataset to that user (the actor pads with related + * users), and upserts with a lowercased profile_url — X handles are + * case-insensitive and stored rows are lowercase. + */ +export async function handleTwitterUserScraperResults(parsed: ApifyWebhookPayload) { + const storeId = parsed.resource.defaultKeyValueStoreId; + if (!storeId) return { social: null }; + + const inputRecord = await apifyClient.keyValueStore(storeId).getRecord("INPUT"); + const input = inputRecord?.value as { twitterHandles?: string[] } | undefined; + const requestedHandle = input?.twitterHandles?.[0]?.trim(); + if (!requestedHandle) return { social: null }; + + const { items } = await apifyClient.dataset(parsed.resource.defaultDatasetId).listItems(); + const user = (items as TwitterUserItem[]).find( + item => item.userName?.toLowerCase() === requestedHandle.toLowerCase(), + ); + if (!user?.url) return { social: null }; + + const social = { + profile_url: normalizeProfileUrl(user.url).toLowerCase(), + username: user.userName, + avatar: user.profilePicture ?? null, + bio: user.description || null, + followerCount: user.followers ?? null, + followingCount: user.following ?? null, + region: user.location || null, + }; + await upsertSocials([social]); + + return { social }; +} diff --git a/lib/apify/twitter/startTwitterUserScraping.ts b/lib/apify/twitter/startTwitterUserScraping.ts new file mode 100644 index 00000000..ee1da451 --- /dev/null +++ b/lib/apify/twitter/startTwitterUserScraping.ts @@ -0,0 +1,37 @@ +import apifyClient from "@/lib/apify/client"; +import { getApifyWebhooks } from "@/lib/apify/getApifyWebhooks"; +import { ApifyRunInfo } from "@/lib/apify/types"; + +/** + * Starts a profile-level X/Twitter scrape via `apidojo/twitter-user-scraper` + * (resolved actor id `V38PZzpEgOfeeWvZY`). Used as the fallback when the + * tweet-based `apidojo/twitter-scraper-lite` run returns zero items — a + * tweetless account never surfaces its `author` profile stats on tweet items, + * so its connected social row would otherwise never update (chat#1851). + * + * Results are persisted by `handleTwitterUserScraperResults` via the shared + * Apify webhook receiver. + */ +const startTwitterUserScraping = async (handle: string): Promise => { + const cleanHandle = handle.trim(); + + if (!cleanHandle) { + throw new Error("Invalid Twitter handle"); + } + + const input = { + twitterHandles: [cleanHandle], + maxItems: 1, + }; + + const run = await apifyClient + .actor("apidojo/twitter-user-scraper") + .start(input, { webhooks: getApifyWebhooks() }); + + return { + runId: run.id, + datasetId: run.defaultDatasetId, + }; +}; + +export default startTwitterUserScraping; diff --git a/lib/apify/validateApifyWebhookRequest.ts b/lib/apify/validateApifyWebhookRequest.ts index 2a446b59..d1d9f43c 100644 --- a/lib/apify/validateApifyWebhookRequest.ts +++ b/lib/apify/validateApifyWebhookRequest.ts @@ -14,6 +14,10 @@ export const apifyWebhookPayloadSchema = z.object({ }), resource: z.object({ defaultDatasetId: z.string().min(1, "resource.defaultDatasetId is required"), + // Present on Apify's default run payload; needed by the X tweetless + // fallback to read the run INPUT (chat#1851). Optional so replayed or + // trimmed payloads keep validating. + defaultKeyValueStoreId: z.string().optional(), }), });