fix(harvest): reprobe newly-discovered slugs on the next pass

scraper/src/harvest/runner.test.ts

@@ -146,6 +146,29 @@ describe("runHarvest", () => {
     expect(result.tenants.every((t) => t.status === "transient_failure")).toBe(true);
   });
 
+  it("stamps skipProbe new slugs with an epoch last_probed_at so the next reprobe picks them up", async () => {
+    const fetchFn = mock(async (input: Request | string) => {
+      const url = typeof input === "string" ? input : input.url;
+      if (url.includes("showNumPages")) return new Response("1", { status: 200 });
+      return new Response(FAKE_CDX, { status: 200 });
+    });
+    const result = await runHarvest({
+      ats: "greenhouse",
+      snapshots: ["2026-13"],
+      client: clientWith(fetchFn),
+      observedAt: OBSERVED_AT,
+      skipProbe: true,
+    });
+    expect(result.tenants.length).toBeGreaterThan(0);
+    for (const t of result.tenants) {
+      // A never-probed slug must NOT masquerade as freshly probed: last_probed_at
+      // is the epoch sentinel (maximally stale) so the reprobe's max-age gate
+      // includes it immediately, while first_seen_at is genuinely today.
+      expect(t.last_probed_at).toBe("1970-01-01T00:00:00.000Z");
+      expect(t.first_seen_at).toBe(OBSERVED_AT);
+    }
+  });
+
   it("counts cdx_fetch_errors when a CDX request fails (not 404)", async () => {
     const fetchFn = mock(async (input: Request | string) => {
       const url = typeof input === "string" ? input : input.url;

scraper/src/harvest/runner.ts

@@ -12,6 +12,16 @@ import {
 import { harvestPatternFor } from "./patterns.ts";
 import { probeMany } from "./probe.ts";
 
+// A newly-discovered slug in skipProbe mode has never actually been probed,
+// so its last_probed_at is stamped with the Unix epoch — a maximally-stale
+// sentinel. The reprobe's `--max-age-days` gate re-probes rows older than the
+// cutoff, so an epoch row is picked up on the very next reprobe pass instead
+// of waiting out a full staleness window from the discovery timestamp. Using
+// `now` here would make a never-probed slug look freshly probed and leave it
+// invisible to both reprobe (too "fresh") and scrape (not yet `live`) for the
+// whole window. first_seen_at stays the real discovery day.
+const NEVER_PROBED_AT = "1970-01-01T00:00:00.000Z";
+
 export interface HarvestRunOptions {
   readonly ats: ATSId;
   readonly snapshots: ReadonlyArray<string>;
@@ -297,12 +307,14 @@ export async function runHarvest(opts: HarvestRunOptions): Promise<HarvestResult
     }
 
     // skipProbe mode — record the slug as transient_failure pending a
-    // later reprobe pass. first_seen_at is set to today.
+    // later reprobe pass. last_probed_at is the epoch sentinel (it has never
+    // actually been probed) so the next reprobe includes it immediately;
+    // first_seen_at is genuinely today.
     const t: Tenant = {
       ats: opts.ats,
       slug,
       status: "transient_failure" as const,
-      last_probed_at: opts.observedAt,
+      last_probed_at: NEVER_PROBED_AT,
       first_seen_at: opts.observedAt,
     };
     return meta ? { ...t, metadata: meta } : t;