Honor --canonical-url on every network that has one

README.md

@@ -155,7 +155,40 @@ publication (and on your own blog, a commit), so those only post when named in
 `--to`. One post can name several: `--to htmlblog,devto` writes the page and
 the article from the same Markdown. Per-post flags: `--slug`,
 `--description`, `--tags`, `--date` (the future is refused), `--draft true`,
-`--author`, and `--overwrite true` for a Git blog.
+`--author`, `--canonical-url`, and `--overwrite true` for a Git blog.
+
+#### Which copy is the original
+
+`--to htmlblog,devto` publishes the same article twice, so one of them has to
+be the original or search engines pick for you. `--canonical-url` says which:
+
+```sh
+myna post --to devto --title "Release 1.2" \
+  --canonical-url https://example.com/blog/042-post.html < post.md
+```
+
+It is honored wherever the network has a field for it, and the field is
+different every time:
+
+| network | what it sends |
+| --- | --- |
+| dev.to | `canonical_url` |
+| Hashnode | `originalArticleURL` |
+| Ghost | `canonical_url` |
+| Tumblr | `source_url`, the attribution link it has instead |
+| gitblog | `canonical:` in the post's frontmatter, for the site template to render |
+| htmlblog | `<link rel="canonical">` in the page head |
+
+WordPress and Micro.blog are left out on purpose. WordPress core has no
+canonical field (it belongs to an SEO plugin's post meta) and Micropub defines
+no canonical property, so neither pretends to support one.
+
+**htmlblog points at itself** without being asked, using the `siteUrl` you
+logged in with, because the original should confirm what the copies claim.
+Pass `--canonical-url` only when the original really is elsewhere. When
+profullstack/cli-tools' `blog-post` writes the page, it applies its own
+`siteUrl` and myna forwards the flag only when you set one — that needs
+cli-tools 0.28.0 or newer.
 
 ### YouTube: search, then comment
 

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/myna-api",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "private": true,
   "type": "module",
   "scripts": {

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/myna",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "A terminal social media manager. Log in, compose, schedule and post to every network from one TUI.",
   "license": "MIT",
   "type": "module",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/myna-core",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Network adapters, credential vault, scheduling and the AI writer behind myna.",
   "license": "MIT",
   "type": "module",

packages/core/src/net/adapters/blogs.ts

@@ -4,6 +4,13 @@
  * These take a title and a body, so a thread-length post becomes an article
  * rather than being split. All of them authenticate with a key you paste,
  * except WordPress, which has genuine application passwords.
+ *
+ * Syndicating the same article to several of these makes one of them the
+ * original, so `--canonical-url` is honored wherever the network has a field
+ * for it: dev.to `canonical_url`, Hashnode `originalArticleURL`, Ghost
+ * `canonical_url`, Tumblr `source_url`. WordPress core has no such field —
+ * canonical there belongs to an SEO plugin's post meta — and Micropub defines
+ * no canonical property, so neither of those pretends to support it.
  */
 import type { Network, TimelineItem } from "../types.ts";
 import { getJson, normalizeInstance, postJson, request } from "../../util/http.ts";
@@ -112,6 +119,7 @@ export const hashnode: Network = {
             publicationId: input.extra?.publicationId || account.meta.publicationId,
             title: input.title || firstLine(input.text),
             contentMarkdown: input.text,
+            ...(input.extra?.canonicalUrl ? { originalArticleURL: input.extra.canonicalUrl } : {}),
             tags: (input.extra?.tags ?? "")
               .split(",")
               .map((tag) => tag.trim().replace(/^#/, ""))
@@ -167,6 +175,7 @@ export const ghost: Network = {
             title: input.title || firstLine(input.text),
             html: `<p>${input.text.replace(/\n\n/g, "</p><p>").replace(/\n/g, "<br>")}</p>`,
             status: input.extra?.draft === "true" ? "draft" : "published",
+            ...(input.extra?.canonicalUrl ? { canonical_url: input.extra.canonicalUrl } : {}),
             ...(input.extra?.tags ? { tags: input.extra.tags.split(",").map((name) => ({ name: name.trim() })) } : {}),
           },
         ],
@@ -353,6 +362,9 @@ export const tumblr: Network = {
         { type: "text", text: input.text },
       ],
       state: input.extra?.draft === "true" ? "draft" : "published",
+      // Tumblr has no rel=canonical of its own; source_url is the attribution
+      // link it does have, and it points where a canonical would.
+      ...(input.extra?.canonicalUrl ? { source_url: input.extra.canonicalUrl } : {}),
       tags: (input.extra?.tags ?? "").split(",").map((tag) => tag.trim().replace(/^#/, "")).filter(Boolean).join(","),
     };
     // The signature covers only the OAuth parameters when the body is JSON.

packages/core/src/net/adapters/ownblogs.ts

@@ -154,6 +154,9 @@ export function gitblogFile(account: Account, input: PostInput, now: Date = new
     title,
     date: date.toISOString().slice(0, 10),
     description,
+    // The site template has to render this; myna only records it. A repo blog
+    // that syndicates elsewhere needs it, and one that does not ignores it.
+    canonical: input.extra?.canonicalUrl || undefined,
     author: input.extra?.author || account.meta.author || undefined,
     tags: tagList(input.extra?.tags),
     draft: input.extra?.draft === "true" ? true : undefined,
@@ -302,6 +305,17 @@ export const gitblog: Network = {
 
 const POST_FILE = /^(\d+)-post\.html$/;
 
+/**
+ * The address a post file is served at.
+ *
+ * Shared by the link myna reports, the feed's timeline and the canonical tag,
+ * because a canonical that does not byte-match the real URL is a canonical
+ * pointing at a different page.
+ */
+export function postUrl(siteUrl: string, file: string): string {
+  return `${String(siteUrl).replace(/\/+$/, "")}/${file}`;
+}
+
 interface HtmlBlogConfig {
   siteTitle?: string | null;
   author?: string | null;
@@ -334,19 +348,22 @@ export function nextPostNumber(names: string[]): string {
 
 /** A whole page in the shape the plain-HTML blog uses. */
 export function renderHtmlPost(
-  post: { title: string; description: string; date: string; body: string },
+  post: { title: string; description: string; date: string; body: string; canonical?: string },
   config: HtmlBlogConfig = {},
 ): string {
   const day = post.date.slice(0, 10);
   const site = config.siteTitle ? ` — ${escapeHtml(config.siteTitle)}` : "";
   const byline = config.author ? `<p><em>${day}, by ${escapeHtml(config.author)}.</em></p>` : `<p><em>${day}</em></p>`;
   const disclosure = config.disclosure ? `\n\n<p><small>${config.disclosure}</small></p>` : "";
+  // Omitted when there is nothing to point at: a canonical aimed at nowhere is
+  // worse than none, because search engines act on it.
+  const canonical = post.canonical ? `\n<link rel="canonical" href="${escapeHtml(post.canonical)}">` : "";
   return `<!doctype html>
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="viewport" content="width=device-width,initial-scale=1">
-<title>${escapeHtml(post.title)}${site}</title>
+<title>${escapeHtml(post.title)}${site}</title>${canonical}
 <link rel="alternate" type="application/rss+xml" href="feed.xml">
 <meta name="date" content="${escapeHtml(post.date)}">
 <meta name="description" content="${escapeHtml(post.description)}">
@@ -400,21 +417,43 @@ function runQuiet(command: string, args: string[], cwd: string): { ok: boolean;
  * one written by hand. When it is not installed, myna writes the page itself
  * in the same shape, minus what only that config knows.
  */
-function writeWithBlogPost(tool: string, dir: string, post: { title: string; description: string; date: string; body: string }): string {
+function writeWithBlogPost(
+  tool: string,
+  dir: string,
+  post: { title: string; description: string; date: string; body: string },
+  canonical?: string,
+): string {
   const scratch = mkdtempSync(join(tmpdir(), "myna-blog-"));
   const bodyFile = join(scratch, "body.html");
   writeFileSync(bodyFile, post.body);
-  const result = runQuiet(tool, ["new", post.title, "--description", post.description, "--body", bodyFile, "--date", post.date, "--dir", dir], dir);
+  // Only an explicit override is passed. Left alone, blog-post points the page
+  // at itself from its own siteUrl, which it knows and myna would be guessing.
+  // The flag needs cli-tools 0.28.0 or newer.
+  const args = ["new", post.title, "--description", post.description, "--body", bodyFile, "--date", post.date, "--dir", dir];
+  if (canonical) args.push("--canonical", canonical);
+  const result = runQuiet(tool, args, dir);
   if (!result.ok) throw new Error(`blog-post failed: ${result.output || "no output"}`);
   const created = /created\s+(\S+-post\.html)/.exec(result.output);
   if (!created) throw new Error(`blog-post did not say which file it created:\n${result.output}`);
   return created[1];
 }
 
-function writeNatively(dir: string, post: { title: string; description: string; date: string; body: string }): string {
+function writeNatively(
+  dir: string,
+  post: { title: string; description: string; date: string; body: string },
+  siteUrl?: string,
+  canonical?: string,
+): string {
   const file = `${nextPostNumber(readdirSync(dir))}-post.html`;
+  // The file name is only settled here, so a self-canonical can only be built
+  // here. An explicit one wins: it means the original is somewhere else.
+  const href = canonical ?? (siteUrl ? postUrl(siteUrl, file) : undefined);
   // 'wx': two writers that both read the directory would pick the same number.
-  writeFileSync(join(dir, file), renderHtmlPost(post, readBlogConfig(dir)), { flag: "wx" });
+  writeFileSync(
+    join(dir, file),
+    renderHtmlPost({ ...post, ...(href ? { canonical: href } : {}) }, readBlogConfig(dir)),
+    { flag: "wx" },
+  );
   const indexPath = join(dir, "index.html");
   if (existsSync(indexPath)) {
     writeFileSync(indexPath, insertIntoIndex(readFileSync(indexPath, "utf8"), { file, title: post.title, date: post.date }));
@@ -504,9 +543,12 @@ export const htmlblog: Network = {
     if (!description) throw new Error("A post needs a description for the feed. Write a first paragraph, or pass --description.");
     const post = { title, description, date, body: renderMarkdown(markdown) };
 
+    const canonical = input.extra?.canonicalUrl;
     const tool = findOnPath("blog-post");
-    const file = tool ? writeWithBlogPost(tool, dir, post) : writeNatively(dir, post);
-    const url = `${siteUrl}/${file}`;
+    const file = tool
+      ? writeWithBlogPost(tool, dir, post, canonical)
+      : writeNatively(dir, post, siteUrl, canonical);
+    const url = postUrl(siteUrl, file);
 
     if (mirror) {
       const problem = pushMirror(dir, mirror, [file, "index.html", "feed.xml"], `${file.replace(/-post\.html$/, "")}: ${title}`);
@@ -533,7 +575,7 @@ export const htmlblog: Network = {
           handle: account.handle,
           text: title.replace(/<[^>]+>/g, "").replace(/&mdash;/g, "—").trim(),
           createdAt: date,
-          url: `${siteUrl}/${name}`,
+          url: postUrl(siteUrl, name),
         };
       });
   },

packages/core/src/version.ts

@@ -6,4 +6,4 @@
  * binary reporting the wrong version, and nothing would fail. A test asserts
  * this matches the package.json it is published under.
  */
-export const VERSION = "0.11.0";
+export const VERSION = "0.12.0";

packages/core/test/blog-canonical.test.ts

@@ -0,0 +1,157 @@
+/**
+ * `--canonical-url` on the long-form networks that have a field for it.
+ *
+ * Each adapter is driven against a stubbed fetch and the request body is read
+ * back, because the thing worth asserting is the field name on the wire: they
+ * all differ, and getting one wrong fails silently as a post with no canonical
+ * rather than as an error.
+ */
+import { test, expect } from "bun:test";
+import { devto, hashnode, ghost, tumblr } from "../src/net/adapters/blogs.ts";
+import type { Account, PostInput } from "../src/net/types.ts";
+
+const CANONICAL = "https://example.com/~me/blog/042-post.html";
+
+/** Run one post against a stubbed fetch and hand back the request bodies. */
+async function capture(
+  run: () => Promise<unknown>,
+  reply: (url: string) => Response,
+): Promise<Record<string, any>[]> {
+  const bodies: Record<string, any>[] = [];
+  const realFetch = globalThis.fetch;
+  globalThis.fetch = (async (url: string, init?: RequestInit) => {
+    if (init?.body) {
+      const raw = String(init.body);
+      try {
+        bodies.push(JSON.parse(raw) as Record<string, any>);
+      } catch {
+        bodies.push(Object.fromEntries(new URLSearchParams(raw)));
+      }
+    }
+    return reply(String(url));
+  }) as typeof fetch;
+  try {
+    await run();
+  } finally {
+    globalThis.fetch = realFetch;
+  }
+  return bodies;
+}
+
+const account = (network: string, creds: Record<string, string>, meta: Record<string, string> = {}): Account =>
+  ({
+    id: `${network}:me`,
+    network,
+    handle: "me",
+    addedAt: new Date().toISOString(),
+    creds,
+    meta,
+  }) as unknown as Account;
+
+const input = (canonical?: string): PostInput =>
+  ({
+    text: "Hello\n\nA body.",
+    title: "Hello",
+    ...(canonical ? { extra: { canonicalUrl: canonical } } : {}),
+  }) as PostInput;
+
+/* ------------------------------------------------------------------ dev.to --- */
+
+test("dev.to sends canonical_url, and omits it when there is none", async () => {
+  const reply = (): Response => new Response(JSON.stringify({ id: 1, url: "https://dev.to/x" }), { status: 200 });
+  const acct = account("devto", { apiKey: "k" });
+
+  const [withUrl] = await capture(() => devto.post(acct, input(CANONICAL)), reply);
+  expect(withUrl.article.canonical_url).toBe(CANONICAL);
+
+  const [without] = await capture(() => devto.post(acct, input()), reply);
+  expect(without.article).not.toHaveProperty("canonical_url");
+});
+
+/* ---------------------------------------------------------------- hashnode --- */
+
+test("hashnode sends originalArticleURL, its own name for the same thing", async () => {
+  const reply = (): Response =>
+    new Response(JSON.stringify({ data: { publishPost: { post: { id: "1", url: "https://h/x" } } } }), { status: 200 });
+  const acct = account("hashnode", { token: "t" }, { publicationId: "p" });
+
+  const [withUrl] = await capture(() => hashnode.post(acct, input(CANONICAL)), reply);
+  expect(withUrl.variables.input.originalArticleURL).toBe(CANONICAL);
+
+  const [without] = await capture(() => hashnode.post(acct, input()), reply);
+  expect(without.variables.input).not.toHaveProperty("originalArticleURL");
+});
+
+/* ------------------------------------------------------------------- ghost --- */
+
+test("ghost sends canonical_url on the post", async () => {
+  const reply = (): Response =>
+    new Response(JSON.stringify({ posts: [{ id: "1", url: "https://g/x" }] }), { status: 200 });
+  // A syntactically valid admin key: it is split on ":" and hex-decoded to sign.
+  const acct = account("ghost", { adminApiKey: `${"a".repeat(24)}:${"b".repeat(64)}` }, { url: "https://g" });
+
+  const [withUrl] = await capture(() => ghost.post(acct, input(CANONICAL)), reply);
+  expect(withUrl.posts[0].canonical_url).toBe(CANONICAL);
+
+  const [without] = await capture(() => ghost.post(acct, input()), reply);
+  expect(without.posts[0]).not.toHaveProperty("canonical_url");
+});
+
+/* ------------------------------------------------------------------ tumblr --- */
+
+test("tumblr sends source_url, the attribution link it has instead of a canonical", async () => {
+  const reply = (): Response => new Response(JSON.stringify({ response: { id_string: "1" } }), { status: 200 });
+  const acct = account(
+    "tumblr",
+    { consumerKey: "ck", consumerSecret: "cs", token: "t", tokenSecret: "ts" },
+    { blog: "me.tumblr.com" },
+  );
+
+  const [withUrl] = await capture(() => tumblr.post(acct, input(CANONICAL)), reply);
+  expect(withUrl.source_url).toBe(CANONICAL);
+
+  const [without] = await capture(() => tumblr.post(acct, input()), reply);
+  expect(without).not.toHaveProperty("source_url");
+});
+
+/* ------------------------------------------------------------------- shape --- */
+
+test("every long-form network that claims canonical support actually sends it", async () => {
+  // A guard against adding an adapter, documenting canonical support, and
+  // wiring nothing: each of these must put the URL somewhere in its request.
+  const cases: [string, () => Promise<unknown>, (url: string) => Response][] = [
+    [
+      "devto",
+      () => devto.post(account("devto", { apiKey: "k" }), input(CANONICAL)),
+      () => new Response(JSON.stringify({ id: 1, url: "u" }), { status: 200 }),
+    ],
+    [
+      "hashnode",
+      () => hashnode.post(account("hashnode", { token: "t" }, { publicationId: "p" }), input(CANONICAL)),
+      () => new Response(JSON.stringify({ data: { publishPost: { post: { id: "1", url: "u" } } } }), { status: 200 }),
+    ],
+    [
+      "ghost",
+      () =>
+        ghost.post(
+          account("ghost", { adminApiKey: `${"a".repeat(24)}:${"b".repeat(64)}` }, { url: "https://g" }),
+          input(CANONICAL),
+        ),
+      () => new Response(JSON.stringify({ posts: [{ id: "1", url: "u" }] }), { status: 200 }),
+    ],
+    [
+      "tumblr",
+      () =>
+        tumblr.post(
+          account("tumblr", { consumerKey: "ck", consumerSecret: "cs", token: "t", tokenSecret: "ts" }, { blog: "b" }),
+          input(CANONICAL),
+        ),
+      () => new Response(JSON.stringify({ response: { id_string: "1" } }), { status: 200 }),
+    ],
+  ];
+
+  for (const [name, run, reply] of cases) {
+    const bodies = await capture(run, reply);
+    expect(JSON.stringify(bodies), `${name} dropped the canonical URL`).toContain(CANONICAL);
+  }
+});