diff --git a/packages/db/index.js b/packages/db/index.js index fd7a5c6..3f9084b 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -21,3 +21,4 @@ export * as alerts from './src/alerts.js'; export * as social from './src/social.js'; export * as dataset from './src/dataset.js'; export * as traffic from './src/traffic.js'; +export * as removals from './src/removals.js'; diff --git a/packages/db/migrations/20260905233000_feed_removals.sql b/packages/db/migrations/20260905233000_feed_removals.sql new file mode 100644 index 0000000..ba262d1 --- /dev/null +++ b/packages/db/migrations/20260905233000_feed_removals.sql @@ -0,0 +1,30 @@ +-- Feeds removed at the owner's request, and kept out for good. +-- +-- Deleting a feed's rows honours a removal request for exactly as long as it +-- takes discovery or a resubmission to find the same URL again. This table is +-- the memory: every insert path checks it, so a publisher who asked to be +-- taken down stays down without anyone having to remember them. +-- +-- Matching is by host as well as by exact URL. A Substack, a Ghost site or a +-- personal domain is one publisher, and the request was about the publisher, +-- not about one of their several feed URLs. + +create table if not exists feed_removals ( + id text primary key, + -- The URL that was removed, as it appeared in feeds.feed_url. + feed_url text not null unique, + -- Lower-case hostname of that URL, without a leading "www.". Any feed on this + -- host is refused. + host text not null, + -- What the row looked like when it went, for the record. + slug text, + title text, + -- Why, and who asked: a name or address from the request email, or 'operator'. + reason text, + requested_by text, + -- How much went with it. + items_removed integer not null default 0, + created_at text not null +); + +create index if not exists feed_removals_host_idx on feed_removals (host); diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index a724ed6..ae9ec03 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1,6 +1,7 @@ import { clusterKey, dedupeItems, topicSlug } from '@rssamplifier/feed'; import { newId, nowIso } from './client.js'; +import { FeedRemovedError, dropRemoved, isRemovedUrl, removalHost } from './removals.js'; import { topicLabelSql } from './topicLabel.js'; /** @@ -309,6 +310,13 @@ export async function takenSlugs(db, base) { * @returns {Promise<{ id: string, slug: string }>} */ export async function insertFeed(db, feed) { + // A publisher who asked to be taken down stays down, whichever path finds + // them again. Checked here rather than in each caller so no new caller can + // forget it. + if (await isRemovedUrl(db, feed.feed_url)) { + throw new FeedRemovedError(String(feed.feed_url), removalHost(feed.feed_url) ?? ''); + } + const id = newId(); const now = nowIso(); @@ -2626,6 +2634,9 @@ export async function markCrawlSuccess(db, id, feed, itemCount, intervalMinutes * @returns {Promise} rows actually inserted */ export async function insertFeedsBulk(db, feeds) { + // Removed publishers are dropped silently: a bulk import or a discovery run + // has nobody to tell, and "rows actually inserted" already says how many. + feeds = await dropRemoved(db, feeds); if (feeds.length === 0) return 0; const now = nowIso(); diff --git a/packages/db/src/removals.js b/packages/db/src/removals.js new file mode 100644 index 0000000..af25588 --- /dev/null +++ b/packages/db/src/removals.js @@ -0,0 +1,209 @@ +/** + * Removing a feed at its owner's request, and keeping it out. + * + * Two halves. `removeFeed` takes a publisher down: the feed row, its items, + * extracts, links and author association go with it (foreign keys cascade), + * an author record nothing else refers to goes too, and the request is + * written to `feed_removals`. `isRemovedUrl` / `dropRemoved` are the other + * half: the insert paths ask before adding a feed, so discovery, a bulk import + * or a fresh submission cannot bring a removed publisher back. + * + * Matching is by host, not only by exact URL. The request was about the + * publisher, and a publisher has as many feed URLs as their platform offers. + */ + +import { newId, nowIso } from './client.js'; + +/** @typedef {import('@libsql/client').Client} Client */ + +/** Thrown by an insert path when the URL belongs to a removed publisher. */ +export class FeedRemovedError extends Error { + /** + * @param {string} feedUrl + * @param {string} host + */ + constructor(feedUrl, host) { + super(`${host} was removed from the directory at its owner's request`); + this.name = 'FeedRemovedError'; + this.feedUrl = feedUrl; + this.host = host; + } +} + +/** + * The host a removal is keyed on: lower-case, no port, no leading "www.". + * + * @param {string} url + * @returns {string|null} null when the URL does not parse + */ +export function removalHost(url) { + try { + const host = new URL(String(url)).hostname.toLowerCase(); + return host.replace(/^www\./, '') || null; + } catch { + return null; + } +} + +/** + * Is this URL, or anything on its host, removed? + * + * @param {Client} db + * @param {string} feedUrl + * @returns {Promise} + */ +export async function isRemovedUrl(db, feedUrl) { + const host = removalHost(feedUrl); + const { rows } = await db.execute({ + sql: 'select 1 from feed_removals where feed_url = ? or host = ? limit 1', + args: [String(feedUrl), host ?? ''], + }); + return rows.length > 0; +} + +/** + * The subset of `feeds` whose URL is not removed, in one query. + * + * Bulk paths insert hundreds of rows at a time; asking per row would turn one + * round trip into hundreds. Hosts are looked up as a set instead. + * + * @template {{ feed_url: string }} T + * @param {Client} db + * @param {T[]} feeds + * @returns {Promise} + */ +export async function dropRemoved(db, feeds) { + if (feeds.length === 0) return feeds; + const hosts = [...new Set(feeds.map((f) => removalHost(f.feed_url)).filter(Boolean))]; + const urls = feeds.map((f) => String(f.feed_url)); + const removed = new Set(); + + // SQLite's parameter limit is comfortably above the 500-row chunks the bulk + // paths use, but chunk anyway so a larger caller cannot trip it. + const CHUNK = 400; + for (let i = 0; i < Math.max(hosts.length, urls.length); i += CHUNK) { + const h = hosts.slice(i, i + CHUNK); + const u = urls.slice(i, i + CHUNK); + const clauses = []; + const args = []; + if (h.length > 0) { + clauses.push(`host in (${h.map(() => '?').join(', ')})`); + args.push(...h); + } + if (u.length > 0) { + clauses.push(`feed_url in (${u.map(() => '?').join(', ')})`); + args.push(...u); + } + const { rows } = await db.execute({ + sql: `select feed_url, host from feed_removals where ${clauses.join(' or ')}`, + args, + }); + for (const row of rows) { + removed.add(String(row.host)); + removed.add(String(row.feed_url)); + } + } + if (removed.size === 0) return feeds; + return feeds.filter( + (f) => !removed.has(String(f.feed_url)) && !removed.has(removalHost(f.feed_url) ?? '') + ); +} + +/** + * Take a publisher down and remember it. + * + * Every feed on the URL's host is deleted, not just the URL given, because a + * removal request names a publisher. Authors left with no feed are deleted as + * well: an author page with nothing under it is still the person's name on the + * site. Returns what went, so the reply to the requester can say so. + * + * @param {Client} db + * @param {{ feed_url: string, reason?: string|null, requested_by?: string|null }} request + * @returns {Promise<{ host: string, feeds: { id: string, slug: string, feed_url: string, title: string|null, items: number }[], authors_removed: number, already_recorded: boolean }>} + */ +export async function removeFeed(db, request) { + const feedUrl = String(request.feed_url).trim(); + const host = removalHost(feedUrl); + if (!host) throw new Error(`not a URL: ${feedUrl}`); + + const { rows } = await db.execute({ + sql: `select id, slug, feed_url, title, + (select count(*) from feed_items where feed_id = feeds.id) as items + from feeds + where lower(feed_url) like ? or lower(feed_url) like ? or lower(site_url) like ? or lower(site_url) like ?`, + args: [`%://${host}/%`, `%://www.${host}/%`, `%://${host}/%`, `%://www.${host}/%`], + }); + const feeds = rows + .map((r) => ({ + id: String(r.id), + slug: String(r.slug), + feed_url: String(r.feed_url), + title: r.title == null ? null : String(r.title), + items: Number(r.items ?? 0), + })) + // `like` cannot anchor on the host boundary, so confirm each hit properly. + .filter((f) => removalHost(f.feed_url) === host || f.feed_url === feedUrl); + + let authorsRemoved = 0; + for (const feed of feeds) { + const { rows: authorRows } = await db.execute({ + sql: `select author_id from feed_authors where feed_id = ? + and author_id not in (select author_id from feed_authors where feed_id != ?)`, + args: [feed.id, feed.id], + }); + for (const row of authorRows) { + await db.execute({ sql: 'delete from authors where id = ?', args: [String(row.author_id)] }); + authorsRemoved += 1; + } + // Items, extracts, links and the author association cascade from here. + await db.execute({ sql: 'delete from feeds where id = ?', args: [feed.id] }); + } + + const { rows: existing } = await db.execute({ + sql: 'select 1 from feed_removals where feed_url = ? limit 1', + args: [feedUrl], + }); + const alreadyRecorded = existing.length > 0; + if (!alreadyRecorded) { + const first = feeds[0] ?? null; + await db.execute({ + sql: `insert into feed_removals + (id, feed_url, host, slug, title, reason, requested_by, items_removed, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + args: [ + newId(), + feedUrl, + host, + first?.slug ?? null, + first?.title ?? null, + request.reason ?? null, + request.requested_by ?? null, + feeds.reduce((sum, f) => sum + f.items, 0), + nowIso(), + ], + }); + } + + return { host, feeds, authors_removed: authorsRemoved, already_recorded: alreadyRecorded }; +} + +/** + * Every removal on record, newest first. + * + * @param {Client} db + * @returns {Promise<{ feed_url: string, host: string, slug: string|null, reason: string|null, requested_by: string|null, items_removed: number, created_at: string }[]>} + */ +export async function listRemovals(db) { + const { rows } = await db.execute( + 'select feed_url, host, slug, reason, requested_by, items_removed, created_at from feed_removals order by created_at desc' + ); + return rows.map((r) => ({ + feed_url: String(r.feed_url), + host: String(r.host), + slug: r.slug == null ? null : String(r.slug), + reason: r.reason == null ? null : String(r.reason), + requested_by: r.requested_by == null ? null : String(r.requested_by), + items_removed: Number(r.items_removed ?? 0), + created_at: String(r.created_at), + })); +} diff --git a/packages/db/src/remove-feed.js b/packages/db/src/remove-feed.js new file mode 100644 index 0000000..4dabfd5 --- /dev/null +++ b/packages/db/src/remove-feed.js @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * Take a publisher out of the directory at their request, for good. + * + * node packages/db/src/remove-feed.js https://someone.substack.com/feed \ + * --reason "removal request by email 2026-08-23" --by someone@example.com + * + * node packages/db/src/remove-feed.js --list + * + * Runs against TURSO_DATABASE_URL like migrate.js does. Deletes every feed on + * the URL's host with its items, extracts and orphaned author, and records the + * host in feed_removals so discovery and resubmission cannot bring it back. + */ + +import { connect } from './client.js'; +import { listRemovals, removeFeed } from './removals.js'; + +function parse(argv) { + const out = { url: null, reason: null, by: null, list: false }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--list') out.list = true; + else if (arg === '--reason') out.reason = argv[++i] ?? null; + else if (arg === '--by') out.by = argv[++i] ?? null; + else if (arg.startsWith('--')) throw new Error(`unknown option ${arg}`); + else out.url = arg; + } + return out; +} + +async function main() { + const opts = parse(process.argv.slice(2)); + const db = connect(); + + if (opts.list) { + const removals = await listRemovals(db); + if (removals.length === 0) console.log('no removals on record'); + for (const r of removals) { + console.log( + `${r.created_at} ${r.host} ${r.items_removed} items` + + `${r.requested_by ? ` by ${r.requested_by}` : ''}${r.reason ? ` (${r.reason})` : ''}` + ); + } + return; + } + + if (!opts.url) { + console.error('usage: remove-feed.js [--reason TEXT] [--by WHO] | --list'); + process.exit(2); + } + + const result = await removeFeed(db, { + feed_url: opts.url, + reason: opts.reason, + requested_by: opts.by, + }); + if (result.feeds.length === 0) { + console.log(`nothing listed on ${result.host}${result.already_recorded ? '; already on record' : ''}`); + } + for (const feed of result.feeds) { + console.log(`removed /${feed.slug} (${feed.feed_url}): ${feed.items} items`); + } + if (result.authors_removed > 0) console.log(`removed ${result.authors_removed} orphaned author record(s)`); + console.log(`${result.host} is now refused by every insert path`); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/packages/db/test/removals.test.js b/packages/db/test/removals.test.js new file mode 100644 index 0000000..0f3441b --- /dev/null +++ b/packages/db/test/removals.test.js @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, newId, nowIso } from '../src/client.js'; +import { migrate } from '../src/migrate.js'; +import * as q from '../src/queries.js'; +import { + FeedRemovedError, + dropRemoved, + isRemovedUrl, + listRemovals, + removalHost, + removeFeed, +} from '../src/removals.js'; + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-removals-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); + // Cascades are the whole mechanism; a file database has them off by default. + await db.execute('pragma foreign_keys = on'); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function addItem(feedId, url) { + await db.execute({ + sql: `insert into feed_items (id, feed_id, guid, url, title, published_at, created_at) + values (?, ?, ?, ?, ?, ?, ?)`, + args: [newId(), feedId, url, url, 'post', nowIso(), nowIso()], + }); +} + +test('removalHost keys on the bare host', () => { + assert.equal(removalHost('https://WWW.Example.com:443/feed'), 'example.com'); + assert.equal(removalHost('https://judith.substack.com/feed'), 'judith.substack.com'); + assert.equal(removalHost('not a url'), null); +}); + +test('removeFeed deletes the feed, its items and an orphaned author, and records it', async () => { + const { id } = await q.insertFeed(db, { + slug: 'potter-substack-com', + feed_url: 'https://potter.substack.com/feed', + site_url: 'https://potter.substack.com', + title: 'Potter', + }); + await addItem(id, 'https://potter.substack.com/p/one'); + await addItem(id, 'https://potter.substack.com/p/two'); + + const authorId = newId(); + await db.execute({ + sql: 'insert into authors (id, slug, identity_key, name, norm_name, created_at, updated_at) values (?, ?, ?, ?, ?, ?, ?)', + args: [authorId, 'potter', 'potter', 'Potter', 'potter', nowIso(), nowIso()], + }); + await db.execute({ + sql: 'insert into feed_authors (feed_id, author_id, created_at) values (?, ?, ?)', + args: [id, authorId, nowIso()], + }); + + const result = await removeFeed(db, { + feed_url: 'https://potter.substack.com/feed', + reason: 'owner asked', + requested_by: 'potter@example.com', + }); + + assert.equal(result.host, 'potter.substack.com'); + assert.deepEqual( + result.feeds.map((f) => [f.slug, f.items]), + [['potter-substack-com', 2]] + ); + assert.equal(result.authors_removed, 1); + assert.equal(result.already_recorded, false); + + assert.equal(await q.feedBySlug(db, 'potter-substack-com'), null); + const { rows: items } = await db.execute({ + sql: 'select count(*) as n from feed_items where feed_id = ?', + args: [id], + }); + assert.equal(Number(items[0].n), 0); + const { rows: authors } = await db.execute({ + sql: 'select count(*) as n from authors where id = ?', + args: [authorId], + }); + assert.equal(Number(authors[0].n), 0); + + const removals = await listRemovals(db); + assert.equal(removals.length, 1); + assert.equal(removals[0].host, 'potter.substack.com'); + assert.equal(removals[0].items_removed, 2); + assert.equal(removals[0].requested_by, 'potter@example.com'); +}); + +test('a removed host stays out of every insert path', async () => { + assert.equal(await isRemovedUrl(db, 'https://potter.substack.com/feed'), true); + // Another URL on the same publisher's host is the same request. + assert.equal(await isRemovedUrl(db, 'https://www.potter.substack.com/feed?format=rss'), true); + assert.equal(await isRemovedUrl(db, 'https://someone-else.substack.com/feed'), false); + + await assert.rejects( + q.insertFeed(db, { + slug: 'potter-again', + feed_url: 'https://potter.substack.com/feed', + title: 'Potter again', + }), + FeedRemovedError + ); + + const inserted = await q.insertFeedsBulk(db, [ + { + slug: 'potter-bulk', + feed_url: 'https://potter.substack.com/feed?bulk=1', + title: 'Potter bulk', + next_fetch_at: nowIso(), + }, + { + slug: 'fine-blog', + feed_url: 'https://fine.example/feed', + title: 'Fine', + next_fetch_at: nowIso(), + }, + ]); + assert.equal(inserted, 1, 'only the feed that was not removed went in'); + assert.equal(await q.feedBySlug(db, 'potter-bulk'), null); + assert.ok(await q.feedBySlug(db, 'fine-blog')); + + const kept = await dropRemoved(db, [ + { feed_url: 'https://potter.substack.com/rss' }, + { feed_url: 'https://fine.example/feed' }, + ]); + assert.deepEqual( + kept.map((f) => f.feed_url), + ['https://fine.example/feed'] + ); +}); + +test('removing again is idempotent and reports nothing to delete', async () => { + const result = await removeFeed(db, { feed_url: 'https://potter.substack.com/feed' }); + assert.deepEqual(result.feeds, []); + assert.equal(result.already_recorded, true); + assert.equal((await listRemovals(db)).length, 1); +}); + +test('removeFeed refuses something that is not a URL', async () => { + await assert.rejects(removeFeed(db, { feed_url: 'nope' }), /not a URL/); +}); diff --git a/packages/ingest/src/keywords.js b/packages/ingest/src/keywords.js index baab052..4c9d809 100644 --- a/packages/ingest/src/keywords.js +++ b/packages/ingest/src/keywords.js @@ -151,14 +151,17 @@ export async function checkCandidate(db, candidate, opts = {}) { } catch (err) { // Another run inserting the same feed between the lookup and the insert is // a duplicate, not a failure. - const raced = await q.feedByUrl(db, feedUrl); + // A publisher who asked to be removed is a rejection with a reason, not a + // fault in the run: the candidate was fine, the answer is still no. + const removed = err?.name === 'FeedRemovedError'; + const raced = removed ? null : await q.feedByUrl(db, feedUrl); await discovery.markCandidate(db, String(candidate.id), { - status: raced ? 'rejected' : 'error', + status: raced || removed ? 'rejected' : 'error', feedUrl, slug: raced ? String(raced.slug) : null, - reason: raced ? 'already-indexed' : String(err?.message ?? err), + reason: raced ? 'already-indexed' : removed ? 'removed-by-owner' : String(err?.message ?? err), }); - return { status: raced ? 'rejected' : 'error' }; + return { status: raced || removed ? 'rejected' : 'error' }; } await q.upsertItems(db, inserted.id, feed.items);