Stop the autoblog spam: every subject, none researched alone

app/(app)/dashboard/autoblog/crawler/page.tsx

@@ -0,0 +1,168 @@
+// Live view of the feed crawler, modelled on rssamplifier.com/crawlstats.
+//
+// The autoblog cites real posts from RSS Amplifier topic feeds, and until the
+// crawl was daemonized that source was invisible: a topic going missing looked
+// identical to a topic nobody had configured. This is where that difference
+// becomes visible.
+//
+// Deliberately read-only and unconfigurable. The feed list is derived from
+// every active site's master keywords on each sweep, so there is nothing here
+// to add or remove — a subject added to a blog appears within a tick. An
+// "add a feed" button would imply a curation step that does not exist, and
+// would create a queue of requests waiting for a human, which is the thing
+// the automation is meant to remove.
+
+import { serviceClient } from "@/lib/supabase/service";
+import { ago, loadCrawlerStats } from "@/lib/lx/feedCrawlStats";
+
+// Always fresh: a status page served from a cache reports the cache's health,
+// not the crawler's, and would go on saying "healthy" through an outage.
+export const dynamic = "force-dynamic";
+
+function Stat({
+  label,
+  value,
+  note,
+}: {
+  label: string;
+  value: string;
+  note?: string;
+}) {
+  return (
+    <div className="card p-4">
+      <div className="text-2xl font-bold tabular-nums">{value}</div>
+      <div className="text-xs uppercase tracking-wider text-[var(--color-muted)]">
+        {label}
+      </div>
+      {note ? (
+        <div className="mt-1 text-xs text-[var(--color-muted)]">{note}</div>
+      ) : null}
+    </div>
+  );
+}
+
+export default async function CrawlerStatusPage() {
+  const { stats, sources } = await loadCrawlerStats(serviceClient());
+  const now = Date.now();
+
+  return (
+    <div className="mx-auto max-w-5xl">
+      <h1 className="text-3xl font-bold">Crawler status</h1>
+      <p className="mt-2 text-[var(--color-muted)]">
+        Live view of the daemon that reads the RSS Amplifier topic feeds the
+        autoblog cites from. The feed list is derived from every active
+        site&apos;s master keywords — nothing here is curated by hand. The same
+        numbers are available as{" "}
+        <a className="underline" href="/api/lx/feed-crawl/status">
+          JSON
+        </a>
+        .
+      </p>
+
+      {stats.stalled ? (
+        <div className="mt-4 rounded border border-[var(--color-danger,#b91c1c)] p-3 text-sm">
+          <strong>Stalled.</strong> {stats.dueNow.toLocaleString()} feed(s) are
+          due and nothing has been read successfully since{" "}
+          {ago(stats.lastSuccessAt, now)}. Either the worker is not running or
+          the directory is unreachable — this page cannot tell which, and the
+          two have different fixes.
+        </div>
+      ) : null}
+
+      <div className="mt-6 grid grid-cols-2 gap-3 sm:grid-cols-4">
+        <Stat
+          label="Feeds"
+          value={stats.sources.toLocaleString()}
+          note={`${stats.active.toLocaleString()} active`}
+        />
+        <Stat
+          label="Due now"
+          value={stats.dueNow.toLocaleString()}
+          note="waiting to be crawled"
+        />
+        <Stat
+          label="Cached posts"
+          value={stats.items.toLocaleString()}
+          note={`${stats.newItems24h.toLocaleString()} new in 24h`}
+        />
+        <Stat
+          label="Erroring"
+          value={stats.erroring.toLocaleString()}
+          note={`${stats.gaveUp.toLocaleString()} given up`}
+        />
+      </div>
+
+      <p className="mt-3 text-xs text-[var(--color-muted)]">
+        Last successful fetch {ago(stats.lastSuccessAt, now)} ·{" "}
+        {stats.neverFetched.toLocaleString()} never fetched · generated{" "}
+        {stats.generatedAt}
+      </p>
+
+      <h2 className="mt-8 text-xl font-semibold">Feeds</h2>
+      <p className="mt-1 text-sm text-[var(--color-muted)]">
+        One row per subject any active blog covers. A feed is re-read at most
+        every 6 hours, least-recently-fetched first, 25 per sweep — so the work
+        per tick is fixed however many subjects the platform grows to.
+      </p>
+
+      {sources.length === 0 ? (
+        <p className="mt-4 text-sm text-[var(--color-muted)]">
+          No feeds yet. The first sweep derives them from the active sites&apos;
+          master keywords; if this stays empty, no site has master keywords set.
+        </p>
+      ) : (
+        <div className="mt-4 overflow-x-auto">
+          <table className="w-full text-sm">
+            <thead>
+              <tr className="text-left text-xs uppercase tracking-wider text-[var(--color-muted)]">
+                <th className="py-2 pr-4">Topic</th>
+                <th className="py-2 pr-4">Status</th>
+                <th className="py-2 pr-4">Last read</th>
+                <th className="py-2 pr-4">Last success</th>
+                <th className="py-2 pr-4 text-right">Items</th>
+                <th className="py-2">Last error</th>
+              </tr>
+            </thead>
+            <tbody>
+              {sources.map((source) => (
+                <tr
+                  key={source.id}
+                  className="border-t border-[var(--color-border,#2a2a2a)]"
+                >
+                  <td className="py-2 pr-4">
+                    <a className="underline" href={source.url} rel="noopener">
+                      {source.topic}
+                    </a>
+                  </td>
+                  <td className="py-2 pr-4">
+                    {source.status === "given_up" ? (
+                      <span
+                        title={`${source.consecutive_failures} consecutive failures`}
+                      >
+                        given up
+                      </span>
+                    ) : source.consecutive_failures > 0 ? (
+                      <span>failing ({source.consecutive_failures})</span>
+                    ) : (
+                      <span>active</span>
+                    )}
+                  </td>
+                  <td className="py-2 pr-4">{ago(source.last_fetch_at, now)}</td>
+                  <td className="py-2 pr-4">
+                    {ago(source.last_success_at, now)}
+                  </td>
+                  <td className="py-2 pr-4 text-right tabular-nums">
+                    {source.item_count.toLocaleString()}
+                  </td>
+                  <td className="py-2 text-xs text-[var(--color-muted)]">
+                    {source.last_error ?? ""}
+                  </td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      )}
+    </div>
+  );
+}

app/(app)/dashboard/projects/[id]/autoblog/setup/form.tsx

@@ -23,8 +23,10 @@ type Existing = {
   target_audiences: string[];
   description: string;
   seed_keywords: string[];
+  master_keywords: string[];
   modifiers: string[];
   preserve_keywords: boolean;
+  ads_enabled: boolean;
   keywords: string[];
   seo_title: string | null;
   seo_description: string | null;
@@ -109,6 +111,14 @@ export function SetupForm({
     (initial?.target_audiences ?? []).join(", "),
   );
   const [description, setDescription] = useState(initial?.description ?? "");
+  const [masterKeywords, setMasterKeywords] = useState(
+    // Falls back to the seed list for any site the backfill has not reached,
+    // so the field is never blank on a project that already has subjects.
+    (initial?.master_keywords?.length
+      ? initial.master_keywords
+      : (initial?.seed_keywords ?? [])
+    ).join(", "),
+  );
   const [seedKeywords, setSeedKeywords] = useState(
     (initial?.seed_keywords ?? []).join(", "),
   );
@@ -118,6 +128,11 @@ export function SetupForm({
   const [preserveKeywords, setPreserveKeywords] = useState<boolean>(
     initial?.preserve_keywords ?? false,
   );
+  // Default ON, matching the column default. A new project is in the network
+  // unless its owner takes the tick out.
+  const [adsEnabled, setAdsEnabled] = useState<boolean>(
+    initial?.ads_enabled ?? true,
+  );
   const [keywords, setKeywords] = useState(
     (initial?.keywords ?? []).join("\n"),
   );
@@ -425,6 +440,13 @@ export function SetupForm({
   // Each line is a CSV row "<keyword>,<volume>". For seed-building and
   // dedupe we only care about the keyword half (everything before the
   // first comma).
+  function masterKeywordsAsArray(): string[] {
+    return masterKeywords
+      .split(",")
+      .map((s) => s.trim())
+      .filter(Boolean);
+  }
+
   function keywordsAsArray(): string[] {
     return keywords
       .split("\n")
@@ -569,9 +591,11 @@ export function SetupForm({
         niche,
         targetAudiences: audiences,
         description,
+        masterKeywords,
         seedKeywords,
         modifiers,
         preserveKeywords,
+        adsEnabled,
         keywords,
         seoTitle,
         seoDescription,
@@ -843,6 +867,34 @@ export function SetupForm({
             onChange={(e) => setDescription(e.target.value)}
           />
         </div>
+        <div>
+          <label className="text-xs uppercase tracking-wider text-[var(--color-muted)]">
+            Master keywords (comma-separated, 3-12 subjects)
+          </label>
+          <input
+            className="input mt-1 w-full font-mono text-sm"
+            type="text"
+            placeholder="peptide, crypto, casino, marijuana, iptv, torrents"
+            value={masterKeywords}
+            onChange={(e) => setMasterKeywords(e.target.value)}
+          />
+          <p className="mt-1 text-xs text-[var(--color-muted)]">
+            The subjects this blog covers, and the list it always rebuilds from.
+            The planner splits every research run evenly across{" "}
+            <strong>all</strong> of them and fills whichever is furthest behind
+            first, so a subject cannot run away with the schedule. Each one is
+            researched crossed with a modifier below — never on its own.{" "}
+            {masterKeywordsAsArray().length > 12 ? (
+              <strong className="text-[var(--color-danger,#b91c1c)]">
+                {masterKeywordsAsArray().length} entered — only the first 12 are
+                kept.
+              </strong>
+            ) : (
+              <>{masterKeywordsAsArray().length} subject(s).</>
+            )}
+          </p>
+        </div>
+
         <div>
           <label className="text-xs uppercase tracking-wider text-[var(--color-muted)]">
             Seed keywords (comma-separated, 1-3 word head terms)
@@ -913,7 +965,25 @@ export function SetupForm({
             <span>
               <span className="font-medium">Preserve keywords</span> — Refetch
               flows skip the seed list and the long-tail list (still refresh
-              niche, audiences, description).
+              niche, audiences, description). Master keywords are always
+              preserved and are unaffected by this.
+            </span>
+          </label>
+        </div>
+
+        <div>
+          <label className="flex items-center gap-2 text-sm">
+            <input
+              type="checkbox"
+              checked={adsEnabled}
+              onChange={(e) => setAdsEnabled(e.target.checked)}
+            />
+            <span>
+              <span className="font-medium">Join the CrawlProof network</span> —
+              published articles carry an ad unit and a short list of related
+              posts from partner blogs and the RSS Amplifier directory. Partner
+              links are followed; directory links are <code>nofollow ugc</code>.
+              An ad slot is created for this project automatically.
             </span>
           </label>
         </div>

app/(app)/dashboard/projects/[id]/autoblog/setup/page.tsx

@@ -7,7 +7,7 @@ import { SetupForm } from "./form";
 export const metadata = { title: "Autoblog · Setup" };
 
 const SITE_COLUMNS =
-  "id, domain, blog_root_url, sitemap_url, niche, target_audiences, description, seed_keywords, modifiers, preserve_keywords, keywords, seo_title, seo_description, tone, competitors, webhook_url, webhook_secret, daily_article_count, publish_days, publish_hour, internal_links_per_article, backlinks_enabled, external_links_per_article, banner_style, status";
+  "id, domain, blog_root_url, sitemap_url, niche, target_audiences, description, seed_keywords, master_keywords, modifiers, preserve_keywords, ads_enabled, keywords, seo_title, seo_description, tone, competitors, webhook_url, webhook_secret, daily_article_count, publish_days, publish_hour, internal_links_per_article, backlinks_enabled, external_links_per_article, banner_style, status";
 
 export default async function AutoblogSetupPage({
   params,

app/actions/linkExchange.ts

@@ -289,6 +289,9 @@ export type SiteInput = {
   niche: string;
   targetAudiences: string;
   description: string;
+  // Comma-separated 3-12 subjects this blog covers. The durable list the
+  // keyword planner allocates evenly across; see lib/lx/topicPlan.
+  masterKeywords: string;
   // Comma-separated 1-3 word head terms for DataForSEO expansion.
   seedKeywords: string;
   // Comma-separated tail terms ("payments", "merchant account") that the
@@ -297,6 +300,9 @@ export type SiteInput = {
   // When true, Refetch flows skip overwriting the keywords text[]. Used
   // after a hand-curated build via seeds × modifiers.
   preserveKeywords: boolean;
+  // Network opt-in: ad units and partner/directory links in published
+  // articles. Default true — see the migration note on lx_site.ads_enabled.
+  adsEnabled: boolean;
   // Comma-separated. parseList() into a text[].
   keywords: string;
   seoTitle: string;
@@ -360,6 +366,12 @@ export async function createOrUpdateSite(
   const seedKeywords = parseList(input.seedKeywords, 50, 40);
   const modifiers = parseList(input.modifiers, 20, 40);
   const preserveKeywords = !!input.preserveKeywords;
+  // The DB constraint caps this at 12; parse to the same number so an
+  // oversized paste is trimmed here rather than rejected as a write error the
+  // form cannot explain.
+  const masterKeywords = parseList(input.masterKeywords, 12, MAX.keyword);
+  // Absent reads as opted in, matching the column default.
+  const adsEnabled = input.adsEnabled !== false;
   // Keywords use parseKeywordRows so each row keeps its `,<volume>` hint.
   // Allow up to ~80 chars per row to fit "keyword phrase here,12345".
   const keywords = parseKeywordRows(input.keywords, MAX.keywords, MAX.keyword + 20);
@@ -433,6 +445,8 @@ export async function createOrUpdateSite(
         target_audiences: audiences,
         description,
         seed_keywords: seedKeywords,
+        master_keywords: masterKeywords,
+        ads_enabled: adsEnabled,
         modifiers,
         preserve_keywords: preserveKeywords,
         keywords,
@@ -489,6 +503,8 @@ export async function createOrUpdateSite(
           target_audiences: audiences,
           description,
           seed_keywords: seedKeywords,
+          master_keywords: masterKeywords,
+          ads_enabled: adsEnabled,
           modifiers,
           preserve_keywords: preserveKeywords,
           keywords,
@@ -534,6 +550,8 @@ export async function createOrUpdateSite(
       target_audiences: audiences,
       description,
       seed_keywords: seedKeywords,
+      master_keywords: masterKeywords,
+      ads_enabled: adsEnabled,
       modifiers,
       preserve_keywords: preserveKeywords,
       keywords,

app/api/lx/feed-crawl/status/route.ts

@@ -0,0 +1,20 @@
+import { NextResponse } from "next/server";
+import { serviceClient } from "@/lib/supabase/service";
+import { loadCrawlerStats } from "@/lib/lx/feedCrawlStats";
+
+export const runtime = "nodejs";
+// A cached status response reports the cache's health rather than the
+// crawler's, and would keep answering "fine" straight through an outage.
+export const dynamic = "force-dynamic";
+
+/**
+ * Counts only.
+ *
+ * The per-feed rows stay on the dashboard page behind a session: this is the
+ * shape a monitor polls, and it has no reason to carry the error strings some
+ * upstream server wrote.
+ */
+export async function GET() {
+  const { stats } = await loadCrawlerStats(serviceClient());
+  return NextResponse.json(stats, { headers: { "cache-control": "no-store" } });
+}