diff --git a/dist/clients/public.js b/dist/clients/public.js index cdbf49c..581960e 100644 --- a/dist/clients/public.js +++ b/dist/clients/public.js @@ -1,6 +1,6 @@ import { load } from "cheerio"; import { CliError } from "../core/errors.js"; -import { getJson } from "../core/http.js"; +import { getJson, publicGet } from "../core/http.js"; function requireUrl(value, optionName) { if (typeof value !== "string" || !value.trim()) throw new CliError("INVALID_ARGUMENT", `${optionName} is required.`, 2, false, { option: optionName }); @@ -16,26 +16,14 @@ function requireUrl(value, optionName) { } function endpoint(baseUrl, pathname) { const url = new URL(requireUrl(baseUrl, "publication URL")); url.pathname = pathname; url.search = ""; url.hash = ""; return url; } async function fetchText(url) { - let response; - try { - response = await fetch(url, { headers: { "user-agent": "nori-substack-cli/0.1", accept: "application/rss+xml, application/xml, text/xml" } }); - } - catch (error) { - throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : String(error)}`, 8, true); - } + const response = await publicGet(url, { "user-agent": "nori-substack-cli/0.1", accept: "application/rss+xml, application/xml, text/xml" }); if (!response.ok) throw new CliError("HTTP_ERROR", `Substack request failed with HTTP ${response.status}.`, 8, response.status >= 500, { status: response.status }); return response.text(); } async function publicationFromHomepage(publicationUrl) { const url = endpoint(publicationUrl, "/"); - let response; - try { - response = await fetch(url, { headers: { "user-agent": "nori-substack-cli/0.1", accept: "text/html" } }); - } - catch (error) { - throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : String(error)}`, 8, true); - } + const response = await publicGet(url, { "user-agent": "nori-substack-cli/0.1", accept: "text/html" }); if (!response.ok) throw new CliError("HTTP_ERROR", `Substack homepage request failed with HTTP ${response.status}.`, 8, response.status >= 500, { status: response.status }); const html = await response.text(); diff --git a/dist/clients/public.js.map b/dist/clients/public.js.map index d77e131..0160f4f 100644 --- a/dist/clients/public.js.map +++ b/dist/clients/public.js.map @@ -1 +1 @@ -{"version":3,"file":"public.js","sourceRoot":"","sources":["../../src/clients/public.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAG1C,SAAS,UAAU,CAAC,KAAc,EAAE,UAAkB;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,GAAG,UAAU,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACvJ,IAAI,CAAC;QAAC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,KAAK,EAAE,CAAC;QAAC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IAAC,CAAC;IAClH,MAAM,CAAC;QAAC,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,GAAG,UAAU,qCAAqC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAAC,CAAC;AACzI,CAAC;AACD,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAgB,IAAS,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;AAE/L,KAAK,UAAU,SAAS,CAAC,GAAQ;IAC/B,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,EAAE,gDAAgD,EAAE,EAAE,CAAC,CAAC;IAAC,CAAC;IACtJ,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAAC,CAAC;IACtJ,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,qCAAqC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpK,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AACD,KAAK,UAAU,uBAAuB,CAAC,cAAsB;IAC3D,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAAC,CAAC;IACjH,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAAC,CAAC;IACtJ,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,8CAA8C,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7K,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAW,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAsB,CAAC;YAC7D,IAAI,QAAQ,CAAC,GAAG,IAAI,OAAO,QAAQ,CAAC,GAAG,KAAK,QAAQ;gBAAE,OAAO,QAAQ,CAAC,GAAG,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM;QAAC,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,iEAAiE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;AACtH,CAAC;AAGD,MAAM,OAAO,YAAY;IACvB,KAAK,CAAC,cAAc,CAAC,cAAsB;QACzC,IAAI,CAAC;YAAC,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAAC,CAAC;QAC9E,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,YAAY,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,GAAG;gBAAE,OAAO,uBAAuB,CAAC,cAAc,CAAC,CAAC;YAC7I,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IACD,KAAK,CAAC,UAAU,CAAC,cAAsB,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3H,KAAK,CAAC,SAAS,CAAC,cAAsB,EAAE,UAA2B,EAAE,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,eAAe,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1N,KAAK,CAAC,OAAO,CAAC,cAAsB,EAAE,MAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,uBAAuB,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1K,KAAK,CAAC,aAAa,CAAC,cAAsB,EAAE,IAAY,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,iBAAiB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtK,KAAK,CAAC,eAAe,CAAC,aAAqB,EAAE,MAAc,EAAE,MAAe,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,+BAA+B,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5Q,KAAK,CAAC,YAAY,CAAC,cAAsB,EAAE,MAAc,EAAE,KAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,kBAAkB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChQ,KAAK,CAAC,UAAU,CAAC,aAAqB,EAAE,MAAc,EAAE,MAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,gBAAgB,kBAAkB,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC;IACxP,KAAK,CAAC,UAAU,CAAC,aAAqB,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5H,KAAK,CAAC,MAAM,CAAC,aAAqB,EAAE,KAAa,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,4BAA4B,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9K,KAAK,CAAC,OAAO,CAAC,cAAsB;QAClC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/T,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7L,CAAC;CACF"} \ No newline at end of file +{"version":3,"file":"public.js","sourceRoot":"","sources":["../../src/clients/public.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAGrD,SAAS,UAAU,CAAC,KAAc,EAAE,UAAkB;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,GAAG,UAAU,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACvJ,IAAI,CAAC;QAAC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,KAAK,EAAE,CAAC;QAAC,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IAAC,CAAC;IAClH,MAAM,CAAC;QAAC,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,GAAG,UAAU,qCAAqC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAAC,CAAC;AACzI,CAAC;AACD,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAgB,IAAS,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;AAE/L,KAAK,UAAU,SAAS,CAAC,GAAQ;IAC/B,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC,CAAC;IAC3I,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,qCAAqC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpK,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AACD,KAAK,UAAU,uBAAuB,CAAC,cAAsB;IAC3D,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IACtG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,8CAA8C,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7K,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAW,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAsB,CAAC;YAC7D,IAAI,QAAQ,CAAC,GAAG,IAAI,OAAO,QAAQ,CAAC,GAAG,KAAK,QAAQ;gBAAE,OAAO,QAAQ,CAAC,GAAG,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM;QAAC,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,iEAAiE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;AACtH,CAAC;AAGD,MAAM,OAAO,YAAY;IACvB,KAAK,CAAC,cAAc,CAAC,cAAsB;QACzC,IAAI,CAAC;YAAC,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAAC,CAAC;QAC9E,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,YAAY,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,GAAG;gBAAE,OAAO,uBAAuB,CAAC,cAAc,CAAC,CAAC;YAC7I,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IACD,KAAK,CAAC,UAAU,CAAC,cAAsB,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3H,KAAK,CAAC,SAAS,CAAC,cAAsB,EAAE,UAA2B,EAAE,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,eAAe,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1N,KAAK,CAAC,OAAO,CAAC,cAAsB,EAAE,MAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,uBAAuB,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1K,KAAK,CAAC,aAAa,CAAC,cAAsB,EAAE,IAAY,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,iBAAiB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtK,KAAK,CAAC,eAAe,CAAC,aAAqB,EAAE,MAAc,EAAE,MAAe,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,+BAA+B,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5Q,KAAK,CAAC,YAAY,CAAC,cAAsB,EAAE,MAAc,EAAE,KAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,kBAAkB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChQ,KAAK,CAAC,UAAU,CAAC,aAAqB,EAAE,MAAc,EAAE,MAAc,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,gBAAgB,kBAAkB,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC;IACxP,KAAK,CAAC,UAAU,CAAC,aAAqB,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5H,KAAK,CAAC,MAAM,CAAC,aAAqB,EAAE,KAAa,IAAsB,OAAO,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,4BAA4B,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9K,KAAK,CAAC,OAAO,CAAC,cAAsB;QAClC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/T,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC;IAC7L,CAAC;CACF"} \ No newline at end of file diff --git a/dist/core/http.d.ts b/dist/core/http.d.ts index 9607518..ee6faf5 100644 --- a/dist/core/http.d.ts +++ b/dist/core/http.d.ts @@ -1,4 +1,6 @@ export interface JsonGetOptions { query?: ReadonlyArray; } +export declare function isCloudflareChallenge(response: Response): boolean; +export declare function publicGet(url: string | URL, headers: Record): Promise; export declare function getJson(input: string | URL, options?: JsonGetOptions): Promise; diff --git a/dist/core/http.js b/dist/core/http.js index e86b72c..d12cff5 100644 --- a/dist/core/http.js +++ b/dist/core/http.js @@ -25,19 +25,55 @@ async function parseJsonResponse(response) { }); } } +const DEFAULT_RETRIES = 2; +const DEFAULT_RETRY_DELAY_MS = 5_000; +function envInteger(name, fallback) { + const value = Number(process.env[name]); + return Number.isInteger(value) && value >= 0 ? value : fallback; +} +// Substack's Cloudflare edge challenges some datacenter IPs outright: every request gets a 403 with +// `cf-mitigated: challenge`, regardless of endpoint or credentials. It is per-IP and usually short-lived. +export function isCloudflareChallenge(response) { + return response.status === 403 && response.headers.get("cf-mitigated")?.toLowerCase() === "challenge"; +} +function transient(response) { + return response.status >= 500 || isCloudflareChallenge(response); +} +// Public GETs are idempotent, so network errors, 5xx responses, and Cloudflare challenges are retried with +// exponential backoff. Override with NORI_SUBSTACK_HTTP_RETRIES and NORI_SUBSTACK_HTTP_RETRY_DELAY_MS. +export async function publicGet(url, headers) { + const retries = envInteger("NORI_SUBSTACK_HTTP_RETRIES", DEFAULT_RETRIES); + const baseDelay = envInteger("NORI_SUBSTACK_HTTP_RETRY_DELAY_MS", DEFAULT_RETRY_DELAY_MS); + for (let attempt = 0;; attempt += 1) { + let response; + let failure; + try { + response = await fetch(url, { method: "GET", headers }); + } + catch (error) { + failure = error; + } + if (response && !transient(response)) + return response; + if (attempt >= retries) { + if (response && isCloudflareChallenge(response)) { + throw new CliError("CLOUDFLARE_CHALLENGE", `Substack's Cloudflare edge challenged this machine's IP (HTTP 403, cf-mitigated: challenge) on ${attempt + 1} attempts. This is an IP-level block, not an auth or endpoint problem; retry later or from another machine.`, 8, true, { status: 403, cfMitigated: "challenge", attempts: attempt + 1 }); + } + if (response) + return response; + throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${failure instanceof Error ? failure.message : "network request failed"}`, 8, true, { attempts: attempt + 1 }); + } + await response?.body?.cancel(); + await new Promise((resolve) => setTimeout(resolve, baseDelay * 2 ** attempt)); + } +} export async function getJson(input, options = {}) { const url = new URL(input); for (const [name, value] of options.query ?? []) { if (value !== undefined) url.searchParams.append(name, String(value)); } - let response; - try { - response = await fetch(url, { method: "GET", headers: { accept: "application/json", "user-agent": USER_AGENT } }); - } - catch (error) { - throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : "network request failed"}`, 8, true); - } + const response = await publicGet(url, { accept: "application/json", "user-agent": USER_AGENT }); if (response.status === 429) { const seconds = retryAfterSeconds(response.headers.get("retry-after")); const details = { status: response.status }; diff --git a/dist/core/http.js.map b/dist/core/http.js.map index 4507576..565d081 100644 --- a/dist/core/http.js.map +++ b/dist/core/http.js.map @@ -1 +1 @@ -{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/core/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,UAAU,GAAG,6EAA6E,CAAC;AAMjG,SAAS,iBAAiB,CAAC,KAAoB;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAkB;IACjD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,+CAA+C,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE;YACnH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS;SAC/D,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAmB,EAAE,UAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;IACpH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACjJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,GAA4B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrE,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,CAAC,iBAAiB,GAAG,OAAO,CAAC;QAC/D,MAAM,IAAI,QAAQ,CAAC,cAAc,EAAE,oCAAoC,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,qCAAqC,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3M,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC"} \ No newline at end of file +{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/core/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,UAAU,GAAG,6EAA6E,CAAC;AAMjG,SAAS,iBAAiB,CAAC,KAAoB;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAkB;IACjD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,QAAQ,CAAC,kBAAkB,EAAE,+CAA+C,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE;YACnH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS;SAC/D,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,SAAS,UAAU,CAAC,IAAY,EAAE,QAAgB;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClE,CAAC;AAED,oGAAoG;AACpG,0GAA0G;AAC1G,MAAM,UAAU,qBAAqB,CAAC,QAAkB;IACtD,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,KAAK,WAAW,CAAC;AACxG,CAAC;AAED,SAAS,SAAS,CAAC,QAAkB;IACnC,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,qBAAqB,CAAC,QAAQ,CAAC,CAAC;AACnE,CAAC;AAED,2GAA2G;AAC3G,uGAAuG;AACvG,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAiB,EAAE,OAA+B;IAChF,MAAM,OAAO,GAAG,UAAU,CAAC,4BAA4B,EAAE,eAAe,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,UAAU,CAAC,mCAAmC,EAAE,sBAAsB,CAAC,CAAC;IAC1F,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,QAA8B,CAAC;QACnC,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;QAChE,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,GAAG,KAAK,CAAC;QAAC,CAAC;QAClC,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YAAE,OAAO,QAAQ,CAAC;QACtD,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;YACvB,IAAI,QAAQ,IAAI,qBAAqB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,QAAQ,CAAC,sBAAsB,EAAE,kGAAkG,OAAO,GAAG,CAAC,6GAA6G,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;YACpV,CAAC;YACD,IAAI,QAAQ;gBAAE,OAAO,QAAQ,CAAC;YAC9B,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,6BAA6B,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;QAChL,CAAC;QACD,MAAM,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAmB,EAAE,UAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;IAChG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,GAA4B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrE,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,CAAC,iBAAiB,GAAG,OAAO,CAAC;QAC/D,MAAM,IAAI,QAAQ,CAAC,cAAc,EAAE,oCAAoC,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,QAAQ,CAAC,YAAY,EAAE,qCAAqC,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3M,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC"} \ No newline at end of file diff --git a/src/clients/public.ts b/src/clients/public.ts index 9715b0d..6184f54 100644 --- a/src/clients/public.ts +++ b/src/clients/public.ts @@ -1,6 +1,6 @@ import { load } from "cheerio"; import { CliError } from "../core/errors.js"; -import { getJson } from "../core/http.js"; +import { getJson, publicGet } from "../core/http.js"; export interface PostListOptions { limit?: number; offset?: number } function requireUrl(value: unknown, optionName: string): string { @@ -11,17 +11,13 @@ function requireUrl(value: unknown, optionName: string): string { function endpoint(baseUrl: string, pathname: string): URL { const url = new URL(requireUrl(baseUrl, "publication URL")); url.pathname = pathname; url.search = ""; url.hash = ""; return url; } async function fetchText(url: URL): Promise { - let response: Response; - try { response = await fetch(url, { headers: { "user-agent": "nori-substack-cli/0.1", accept: "application/rss+xml, application/xml, text/xml" } }); } - catch (error) { throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : String(error)}`, 8, true); } + const response = await publicGet(url, { "user-agent": "nori-substack-cli/0.1", accept: "application/rss+xml, application/xml, text/xml" }); if (!response.ok) throw new CliError("HTTP_ERROR", `Substack request failed with HTTP ${response.status}.`, 8, response.status >= 500, { status: response.status }); return response.text(); } async function publicationFromHomepage(publicationUrl: string): Promise { const url = endpoint(publicationUrl, "/"); - let response: Response; - try { response = await fetch(url, { headers: { "user-agent": "nori-substack-cli/0.1", accept: "text/html" } }); } - catch (error) { throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : String(error)}`, 8, true); } + const response = await publicGet(url, { "user-agent": "nori-substack-cli/0.1", accept: "text/html" }); if (!response.ok) throw new CliError("HTTP_ERROR", `Substack homepage request failed with HTTP ${response.status}.`, 8, response.status >= 500, { status: response.status }); const html = await response.text(); const $ = load(html); diff --git a/src/core/docs.md b/src/core/docs.md index c73eb52..3a172dd 100644 --- a/src/core/docs.md +++ b/src/core/docs.md @@ -17,6 +17,7 @@ Path: @/src/core - Storage state may come from a default owner-only file, an explicit path, or a named base64 environment handoff. - HTTP errors preserve status and retry safety without exposing credentials. +- Public GETs retry network errors, 5xx responses, and Cloudflare challenges (`403` + `cf-mitigated: challenge`) with exponential backoff (`NORI_SUBSTACK_HTTP_RETRIES`, default 2; `NORI_SUBSTACK_HTTP_RETRY_DELAY_MS`, default 5000). A challenge that outlasts the retries surfaces as the retry-safe `CLOUDFLARE_CHALLENGE` error. - JSON output uses stable success/error envelopes. ### Things to Know @@ -24,5 +25,6 @@ Path: @/src/core - Storage-state files must remain mode `0600`. - Only session cookies are forwarded to authenticated requests. - Non-idempotent requests are never blindly retried. +- A Cloudflare challenge is an IP-level block on the calling machine, not an auth or endpoint failure; the CLI reports it and never tries to solve or bypass it. Created and maintained by Nori. diff --git a/src/core/http.ts b/src/core/http.ts index 10d04c9..5d77658 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -28,17 +28,53 @@ async function parseJsonResponse(response: Response): Promise { } } +const DEFAULT_RETRIES = 2; +const DEFAULT_RETRY_DELAY_MS = 5_000; + +function envInteger(name: string, fallback: number): number { + const value = Number(process.env[name]); + return Number.isInteger(value) && value >= 0 ? value : fallback; +} + +// Substack's Cloudflare edge challenges some datacenter IPs outright: every request gets a 403 with +// `cf-mitigated: challenge`, regardless of endpoint or credentials. It is per-IP and usually short-lived. +export function isCloudflareChallenge(response: Response): boolean { + return response.status === 403 && response.headers.get("cf-mitigated")?.toLowerCase() === "challenge"; +} + +function transient(response: Response): boolean { + return response.status >= 500 || isCloudflareChallenge(response); +} + +// Public GETs are idempotent, so network errors, 5xx responses, and Cloudflare challenges are retried with +// exponential backoff. Override with NORI_SUBSTACK_HTTP_RETRIES and NORI_SUBSTACK_HTTP_RETRY_DELAY_MS. +export async function publicGet(url: string | URL, headers: Record): Promise { + const retries = envInteger("NORI_SUBSTACK_HTTP_RETRIES", DEFAULT_RETRIES); + const baseDelay = envInteger("NORI_SUBSTACK_HTTP_RETRY_DELAY_MS", DEFAULT_RETRY_DELAY_MS); + for (let attempt = 0; ; attempt += 1) { + let response: Response | undefined; + let failure: unknown; + try { response = await fetch(url, { method: "GET", headers }); } + catch (error) { failure = error; } + if (response && !transient(response)) return response; + if (attempt >= retries) { + if (response && isCloudflareChallenge(response)) { + throw new CliError("CLOUDFLARE_CHALLENGE", `Substack's Cloudflare edge challenged this machine's IP (HTTP 403, cf-mitigated: challenge) on ${attempt + 1} attempts. This is an IP-level block, not an auth or endpoint problem; retry later or from another machine.`, 8, true, { status: 403, cfMitigated: "challenge", attempts: attempt + 1 }); + } + if (response) return response; + throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${failure instanceof Error ? failure.message : "network request failed"}`, 8, true, { attempts: attempt + 1 }); + } + await response?.body?.cancel(); + await new Promise((resolve) => setTimeout(resolve, baseDelay * 2 ** attempt)); + } +} + export async function getJson(input: string | URL, options: JsonGetOptions = {}): Promise { const url = new URL(input); for (const [name, value] of options.query ?? []) { if (value !== undefined) url.searchParams.append(name, String(value)); } - let response: Response; - try { - response = await fetch(url, { method: "GET", headers: { accept: "application/json", "user-agent": USER_AGENT } }); - } catch (error) { - throw new CliError("NETWORK_ERROR", `Unable to reach Substack: ${error instanceof Error ? error.message : "network request failed"}`, 8, true); - } + const response = await publicGet(url, { accept: "application/json", "user-agent": USER_AGENT }); if (response.status === 429) { const seconds = retryAfterSeconds(response.headers.get("retry-after")); const details: Record = { status: response.status }; diff --git a/tests/public-retry.test.ts b/tests/public-retry.test.ts new file mode 100644 index 0000000..06912cf --- /dev/null +++ b/tests/public-retry.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, test } from "vitest"; +import { runCli, withHttpServer } from "./helpers.js"; + +const closers: Array<() => Promise> = []; +afterEach(async () => { while (closers.length) await closers.pop()?.(); }); + +const fastRetries = { NORI_SUBSTACK_HTTP_RETRIES: "2", NORI_SUBSTACK_HTTP_RETRY_DELAY_MS: "0" }; + +function challenge(response: import("node:http").ServerResponse): void { + response.statusCode = 403; + response.setHeader("cf-mitigated", "challenge"); + response.setHeader("content-type", "text/html"); + response.end("Just a moment..."); +} + +describe("public read retries", () => { + test("retries a Cloudflare challenge and succeeds once it clears", async () => { + let requests = 0; + const server = await withHttpServer((_request, response) => { + requests += 1; + if (requests === 1) return challenge(response); + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ id: 42, name: "Carbon" })); + }); + closers.push(server.close); + + const result = await runCli(["publication", "get", "--url", server.origin], fastRetries); + + expect(result.code).toBe(0); + expect(requests).toBe(2); + expect(JSON.parse(result.stdout).data).toMatchObject({ id: 42 }); + }); + + test("reports a persistent Cloudflare challenge as a typed retry-safe IP block", async () => { + let requests = 0; + const server = await withHttpServer((_request, response) => { requests += 1; challenge(response); }); + closers.push(server.close); + + const result = await runCli(["publication", "get", "--url", server.origin], fastRetries); + + expect(result.code).toBe(8); + expect(requests).toBe(3); + expect(JSON.parse(result.stderr)).toMatchObject({ + ok: false, + error: { code: "CLOUDFLARE_CHALLENGE", retrySafe: true, status: 403, cfMitigated: "challenge", attempts: 3 }, + }); + }); + + test("retries 5xx responses on public reads", async () => { + let requests = 0; + const server = await withHttpServer((_request, response) => { + requests += 1; + if (requests < 3) { response.statusCode = 503; response.end("unavailable"); return; } + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify([{ id: 7 }])); + }); + closers.push(server.close); + + const result = await runCli(["post", "list", "--publication-url", server.origin], fastRetries); + + expect(result.code).toBe(0); + expect(requests).toBe(3); + }); + + test("does not retry a plain 403 without a Cloudflare challenge", async () => { + const paths: string[] = []; + const server = await withHttpServer((request, response) => { + paths.push(request.url ?? ""); + response.statusCode = 403; + response.end("Not authorized"); + }); + closers.push(server.close); + + const result = await runCli(["post", "list", "--publication-url", server.origin], fastRetries); + + expect(result.code).toBe(8); + expect(paths).toHaveLength(1); + expect(JSON.parse(result.stderr)).toMatchObject({ error: { code: "HTTP_ERROR", status: 403 } }); + }); +});