Tell a construction from a keyword when cleaning up after the fact

lib/lx/keywordsResearch.ts

@@ -37,6 +37,7 @@ import {
   crossQueries,
   dropDuplicates,
   isOnNiche,
+  ownAnchorTokens,
   resolveMasters,
   resolveModifiers,
   signature,
@@ -77,8 +78,14 @@ const MAX_BUYER_JOURNEY_VOLUME_LOOKUP = 160;
  */
 const CROSS_PER_MASTER = 3;
 
-/** A candidate with the subject it belongs to. */
-type Candidate = { row: DfsKeywordRow; master: string };
+/**
+ * A candidate with the subject it belongs to.
+ *
+ * `fromCross` marks the locally-built subject x modifier constructions. They
+ * are excellent *seeds* — that is their real job — and mediocre *articles*, so
+ * they are tracked separately and only published as a last resort.
+ */
+type Candidate = { row: DfsKeywordRow; master: string; fromCross?: boolean };
 
 function parseStoredKeyword(row: string): DfsKeywordRow | null {
   const idx = row.indexOf(",");
@@ -271,6 +278,13 @@ export type KeywordResearchResult = {
   apiCost: number;
   /** Rows allocated per subject — surfaced so a skewed queue is visible. */
   perMaster?: Record<string, number>;
+  /**
+   * True when nothing researched survived and the run fell back to
+   * constructed subject x modifier topics. A degraded result, not a failure —
+   * reported rather than swallowed, because a site sitting on this for weeks
+   * means its upstreams or its modifiers need attention.
+   */
+  usedCrossFloor?: boolean;
   error?: string;
 };
 
@@ -303,6 +317,9 @@ export async function researchKeywords(
   // halves of the gate with one token. See anchorTokens.
   const modifiers = resolveModifiers(site, masters);
   const anchors = anchorTokens(site, masters);
+  // Site-supplied anchors only. A partial match on a multi-word subject may
+  // not be rescued by the generic vocabulary — see isOnNiche.
+  const ownAnchors = ownAnchorTokens(site, masters);
 
   if (masters.length === 0) {
     return {
@@ -376,6 +393,7 @@ export async function researchKeywords(
   const crosses = crossQueries(masters, modifiers, CROSS_PER_MASTER);
   for (const { master, query } of crosses) {
     candidates.push({
+      fromCross: true,
       row: {
         keyword: query,
         search_volume: null,
@@ -487,7 +505,9 @@ export async function researchKeywords(
   // ------------------------------------------------------------------
   // Gate, rank, allocate.
   // ------------------------------------------------------------------
-  const onNiche = candidates.filter((c) => isOnNiche(c.row.keyword, c.master, anchors));
+  const onNiche = candidates.filter((c) =>
+    isOnNiche(c.row.keyword, c.master, anchors, ownAnchors),
+  );
 
   // Volume filtering applies only to what came back from an API with a volume
   // attached. The locally-built crosses have no volume by construction and
@@ -524,20 +544,53 @@ export async function researchKeywords(
     publishedSignatures,
   );
 
-  // Take each subject's allocated share, then interleave so the published
-  // sequence alternates subjects rather than running one to exhaustion.
-  const byMaster = new Map<string, Candidate[]>();
-  for (const master of masters) byMaster.set(master, []);
-  for (const candidate of deduped) {
-    const bucket = byMaster.get(candidate.master);
-    if (!bucket) continue;
-    if (bucket.length >= (allocation.get(candidate.master) ?? 0)) continue;
-    bucket.push({ row: candidate.row, master: candidate.master });
+  /**
+   * Take each subject's allocated share, then interleave so the published
+   * sequence alternates subjects rather than running one to exhaustion.
+   */
+  function select(pool: Candidate[]): Candidate[] {
+    const byMaster = new Map<string, Candidate[]>();
+    for (const master of masters) byMaster.set(master, []);
+    for (const candidate of pool) {
+      const bucket = byMaster.get(candidate.master);
+      if (!bucket) continue;
+      if (bucket.length >= (allocation.get(candidate.master) ?? 0)) continue;
+      bucket.push(candidate);
+    }
+    // Subjects that could not fill their share hand it back, so a subject
+    // with no available candidates costs the run coverage rather than volume.
+    return interleave(byMaster).slice(0, TARGET_KEYWORDS);
   }
 
-  // Subjects that could not fill their share hand it back, so a subject with
-  // no available candidates costs the run coverage rather than volume.
-  const chosen = interleave(byMaster).slice(0, TARGET_KEYWORDS);
+  // The crosses are held back from the normal pass.
+  //
+  // They were being used as per-subject filler, and the first production run
+  // showed what that publishes: "saving money pricing", "deals platform",
+  // "coordination d0rz", "ai content loop". On-niche, gate-passing, and not
+  // article topics anybody would search for — bl0ggers' niche is
+  // "human-in-the-loop AI publishing", so its derived modifiers are literally
+  // "human" and "loop", and crossing a subject with those yields nonsense.
+  //
+  // Their real value is upstream: as DataForSEO seeds they are what turns
+  // "peptide" into "peptide merchant account". So they still seed every
+  // expansion — they just no longer get published on the strength of being
+  // grammatically adjacent to the niche.
+  let chosen = select(deduped.filter((c) => !c.fromCross));
+
+  // Last resort, and site-level rather than per-subject: only when the entire
+  // run would otherwise insert nothing does the constructed floor get used.
+  // That preserves "a blog with every upstream down still publishes" without
+  // letting constructions pad an otherwise healthy run.
+  let usedCrossFloor = false;
+  if (chosen.length === 0) {
+    chosen = select(deduped);
+    usedCrossFloor = chosen.length > 0;
+    if (usedCrossFloor) {
+      console.warn(
+        `[lx] ${site.domain}: no researched keywords survived; falling back to ${chosen.length} constructed subject x modifier topics`,
+      );
+    }
+  }
 
   if (chosen.length === 0) {
     const details = sourceErrors.length > 0
@@ -593,5 +646,11 @@ export async function researchKeywords(
     perMaster[row.master_keyword] = (perMaster[row.master_keyword] ?? 0) + 1;
   }
 
-  return { ok: true, inserted: insertRows.length, apiCost: totalCost, perMaster };
+  return {
+    ok: true,
+    inserted: insertRows.length,
+    apiCost: totalCost,
+    perMaster,
+    usedCrossFloor,
+  };
 }

lib/lx/topicPlan.ts

@@ -223,6 +223,29 @@ export function resolveModifiers(
  * the very same word, satisfying a two-part test with one token. The anchor
  * has to be evidence the subject match did not already provide.
  */
+export function ownAnchorTokens(
+  site: SiteTopicFields,
+  masters: string[] = [],
+): Set<string> {
+  const masterTokens = new Set(masters.flatMap((m) => tokens(m).map(stem)));
+  const defaults = new Set(DEFAULT_MODIFIERS.flatMap((m) => tokens(m).map(stem)));
+  const out = new Set<string>();
+
+  const add = (phrase: string) => {
+    for (const token of tokens(phrase)) {
+      const stemmed = stem(token);
+      if (!masterTokens.has(stemmed) && !defaults.has(stemmed)) out.add(stemmed);
+    }
+  };
+
+  const explicit = (site.modifiers ?? [])
+    .map((m) => (m ?? "").trim())
+    .filter((m) => m.length > 0);
+  for (const modifier of explicit) add(modifier);
+  add(site.niche ?? "");
+  return out;
+}
+
 export function anchorTokens(
   site: SiteTopicFields,
   masters: string[] = [],
@@ -274,6 +297,7 @@ export function isOnNiche(
   keyword: string,
   master: string,
   anchors: Set<string>,
+  ownAnchors?: Set<string>,
 ): boolean {
   const candidate = new Set(tokens(keyword).map(stem));
   if (candidate.size === 0) return false;
@@ -284,6 +308,27 @@ export function isOnNiche(
   const hits = masterTokens.filter((t) => candidate.has(t)).length;
   if (hits === 0) return false;
 
+  // A PARTIAL match on a multi-word subject may only be rescued by an anchor
+  // the site itself supplied — never by the generic commercial vocabulary.
+  //
+  // The first production run leaked exactly this shape: "open standards"
+  // matched on "open" alone and the default anchor "software" then admitted
+  // "open broadcaster software"; "supply chain security" matched on "supply"
+  // and "automation" admitted "industrial automation supply"; "online
+  // shopping" matched on "online" and "pricing" admitted "quickbooks online
+  // pricing". One generic subject word plus one generic tail word is not
+  // evidence of anything, and the defaults are generic by construction.
+  //
+  // A complete subject match is unaffected — including a single-word subject,
+  // where "iptv" + "alternatives" is a real query.
+  if (ownAnchors && masterTokens.length > 1 && hits < masterTokens.length) {
+    if (ownAnchors.size === 0) return false;
+    for (const token of candidate) {
+      if (ownAnchors.has(token)) return true;
+    }
+    return false;
+  }
+
   // A COMPLETE match on a multi-word subject is its own evidence, and needs no
   // anchor.
   //

scripts/purge-constructed-keywords.ts

@@ -0,0 +1,136 @@
+// Remove the two classes the first production run got wrong.
+//
+// Companion to purge-offniche-keywords.ts, for rows that pipeline wrote before
+// PR #213 tightened it. Two verdicts, both computed with the real code rather
+// than approximated in SQL:
+//
+//   1. **Partial-match leaks** — now rejected by `isOnNiche` once it is given
+//      the site's own anchors, so this just re-runs the gate.
+//   2. **Constructions** — a keyword that IS a `subject x modifier` cross.
+//      These pass the gate (they are on-niche by construction) and cannot be
+//      found by a volume check, because the buyer-journey model also returns
+//      keywords with no volume and those are the best output in the run.
+//      Identified by fingerprint against the cross set the planner would build
+//      for that site, which is exact.
+//
+// Prints SQL rather than executing it. Only `queued` rows are ever considered.
+//
+// Usage: npx tsx scripts/purge-constructed-keywords.ts <dump.json>
+
+import { readFileSync } from "node:fs";
+import {
+  anchorTokens,
+  crossQueries,
+  isOnNiche,
+  ownAnchorTokens,
+  resolveMasters,
+  resolveModifiers,
+  signature,
+  stem,
+  tokens,
+} from "../lib/lx/topicPlan";
+
+type Row = {
+  id: string;
+  keyword: string;
+  master_keyword: string | null;
+  domain: string;
+  niche: string | null;
+  master_keywords: string[] | null;
+  modifiers: string[] | null;
+};
+
+function attribute(keyword: string, masters: string[]): string | null {
+  let best: string | null = null;
+  let bestLen = 0;
+  const candidate = new Set(tokens(keyword).map(stem));
+  for (const master of masters) {
+    const masterTokens = tokens(master).map(stem);
+    if (masterTokens.length === 0) continue;
+    const hit = masterTokens.filter((t) => candidate.has(t));
+    if (hit.length === 0) continue;
+    const len = hit.join("").length;
+    if (len > bestLen) {
+      bestLen = len;
+      best = master;
+    }
+  }
+  return best;
+}
+
+function extractRows(raw: string): Row[] {
+  const slice = raw.slice(raw.indexOf("[{"), raw.lastIndexOf("}]") + 2);
+  const text = slice.includes('\\"') ? slice.replace(/\\"/g, '"') : slice;
+  const parsed = JSON.parse(text);
+  const first = parsed[0];
+  return Array.isArray(first?.payload) ? first.payload : parsed;
+}
+
+const rows: Row[] = extractRows(readFileSync(process.argv[2], "utf8"));
+
+// One cross-fingerprint set per site, built once.
+const crossSigs = new Map<string, Set<string>>();
+function crossesFor(row: Row): Set<string> {
+  const cached = crossSigs.get(row.domain);
+  if (cached) return cached;
+  const masters = resolveMasters(row);
+  // Depth well past the planner's 3, so a construction is caught regardless of
+  // how deep that run happened to go.
+  const built = crossQueries(masters, resolveModifiers(row, masters), 8);
+  const set = new Set(built.map((c) => signature(c.query)));
+  crossSigs.set(row.domain, set);
+  return set;
+}
+
+type Bucket = { keep: string[]; leak: string[]; built: string[]; ids: string[] };
+const bySite = new Map<string, Bucket>();
+
+for (const row of rows) {
+  const masters = resolveMasters(row);
+  const b = bySite.get(row.domain) ?? { keep: [], leak: [], built: [], ids: [] };
+
+  const master = row.master_keyword ?? attribute(row.keyword, masters);
+  const anchors = anchorTokens(row, masters);
+  const own = ownAnchorTokens(row, masters);
+
+  const passes =
+    master !== null && isOnNiche(row.keyword, master, anchors, own);
+  const constructed = crossesFor(row).has(signature(row.keyword));
+
+  if (!passes) {
+    b.leak.push(row.keyword);
+    b.ids.push(row.id);
+  } else if (constructed) {
+    b.built.push(row.keyword);
+    b.ids.push(row.id);
+  } else {
+    b.keep.push(row.keyword);
+  }
+  bySite.set(row.domain, b);
+}
+
+const allIds: string[] = [];
+let k = 0;
+let l = 0;
+let c = 0;
+for (const [domain, b] of Array.from(bySite).sort()) {
+  k += b.keep.length;
+  l += b.leak.length;
+  c += b.built.length;
+  allIds.push(...b.ids);
+  console.log(`\n=== ${domain}: keep ${b.keep.length}, leak ${b.leak.length}, constructed ${b.built.length}`);
+  if (b.leak.length) console.log(`  leaked      : ${b.leak.join(" | ")}`);
+  if (b.built.length) console.log(`  constructed : ${b.built.slice(0, 10).join(" | ")}${b.built.length > 10 ? " | …" : ""}`);
+  if (b.keep.length) console.log(`  keeping     : ${b.keep.slice(0, 8).join(" | ")}${b.keep.length > 8 ? " | …" : ""}`);
+}
+
+console.log(`\n--- keep ${k}, delete ${l + c} (${l} leaked, ${c} constructed) of ${rows.length}`);
+console.log("\n-- SQL:");
+for (let i = 0; i < allIds.length; i += 200) {
+  console.log(
+    `delete from lx_keyword where status='queued' and id in (${allIds
+      .slice(i, i + 200)
+      .map((id) => `'${id}'`)
+      .join(",")});`,
+  );
+}