From d180c73b1cdaf43683e146e97688e0fe88a0c23f Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 11 Sep 2026 21:38:26 +0000 Subject: [PATCH] Open a domain from Traffic, and score which ones have legs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Traffic screen ranked properties by visitors, which is the one number on this fleet that lies: a "visitor" is any hit not classified as a crawler, so the busiest row is routinely the least interesting one. Two things follow from that. A property can now be opened. On Traffic, arrows pick a site and Enter — or a click on the row — opens one domain on its own: its pageviews, visits, humans against bots, AI referrals and arrival channels, next to the burn prorated onto it by both denominators, its ad earnings and spend joined exactly rather than shared out, and the commission when there is one merchant business to attribute it to. Esc, left or 2 comes back. It fires no new request — the whole detail is rebuilt from the snapshot the list was drawn from, so the two can never disagree. And each property carries a risk-to-viral score, as a column on the list and taken apart on the domain screen: score = 100 × viral × (1 − risk/2) viral = momentum .40 + discovery .30 + humanity .20 + money .10 risk = volatility .40 + concentration .30 + bots .20 + unmonetised .10 Arithmetic over numbers already on the screen, never a model, and every component prints the figure behind it. A component with no data is dropped and its weight redistributed rather than counted as a zero; a site whose stats call failed is not scored at all; ~ marks a sample under 25 human visits. s cycles the order, --sort=score starts there. Humanity is the reason for the one new server field. `who=humans` filters the RPC to p_kind='human', so its bot column is zero by construction and a share computed from it would call a 99%-crawler property 100% human. `GET /api/tracker/v1/stats?detail=1` therefore adds the series and a second, unfiltered read for the mix — skipped entirely when the caller already asked for everything, and off by default so `crawlproof stats` pays for neither. /api/ads/v1/earnings gains destination_url per campaign and project_id per slot, which are the only fields that name a domain. Gaps named rather than papered over: the earn rail is a network-wide pool with no project column, so there is no per-domain share to show; and CoinPay commission is only attributable when the merchant account has exactly one business. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QuUkTrofSjQ15j79mRuy4f --- README.md | 12 +- app/(marketing)/docs/cli/page.tsx | 69 ++++ app/api/tracker/v1/stats/route.ts | 13 +- cli/dashboard.ts | 623 +++++++++++++++++++++++++----- cli/index.ts | 15 +- lib/ads/earnings-data.ts | 13 +- lib/dashboard/collect.ts | 30 +- lib/dashboard/roi.ts | 29 ++ lib/dashboard/score.ts | 390 +++++++++++++++++++ lib/dashboard/site.ts | 305 +++++++++++++++ lib/tracker/apiStats.ts | 90 ++++- packages/cli/README.md | 45 ++- packages/cli/src/cli.ts | 25 +- tests/dashboard-score.test.ts | 254 ++++++++++++ tests/dashboard-screens.test.ts | 248 ++++++++++++ tests/dashboard-site.test.ts | 228 +++++++++++ tests/tracker-api-stats.test.ts | 45 ++- 17 files changed, 2320 insertions(+), 114 deletions(-) create mode 100644 lib/dashboard/score.ts create mode 100644 lib/dashboard/site.ts create mode 100644 tests/dashboard-score.test.ts create mode 100644 tests/dashboard-screens.test.ts create mode 100644 tests/dashboard-site.test.ts diff --git a/README.md b/README.md index 20f3bb8..4f91fb0 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ echo '{ "token": "crp_…" }' > ~/.crawlproof.json && chmod 600 ~/.crawlproof.js crawlproof dashboard # five live screens: ROI, Traffic, Ads, Money, Spend crawlproof dashboard --range=1m --json | jq .roi.derived +crawlproof dashboard --json | jq '.sites[] | {site, score: .score.score}' # ranked properties crawlproof stats example.com # sources, referrers, top pages as text crawlproof ad https://example.com/launch crawlproof ads pause crawlproof-ad-144 @@ -154,12 +155,21 @@ esbuild-bundles `lib/dashboard/*` and `cli/dashboard.ts` where they already live and where the test suite already covers them, leaving hqtui and the CoinPay SDK external. Rebuild with `node packages/cli/build.mjs`. +On the Traffic screen, `↑`/`↓` pick a property and `Enter` (or a click) opens +it: that domain's traffic and money on their own, and its **risk-to-viral +score** taken apart into the components it was built from — +`100 × viral × (1 − risk/2)`, where viral is momentum, discovery, humanity and +money, and risk is volatility, channel concentration, bot dependence and being +unmonetised. A component with no data is dropped rather than counted as a zero. +The full table is in [`packages/cli/README.md`](packages/cli/README.md) and the +formula lives in [`lib/dashboard/score.ts`](lib/dashboard/score.ts). + The endpoints underneath are the same ones anything else can call: | Route | Answers | | --- | --- | | `GET /api/tracker/v1/sites` | the projects this token can read | -| `GET /api/tracker/v1/stats?site=&range=&who=` | who arrived and from where | +| `GET /api/tracker/v1/stats?site=&range=&who=&detail=1` | who arrived and from where; `detail=1` adds the series and the unfiltered human / bot mix | | `GET /api/ads/v1/earnings?days=` | ad delivery and money, both sides | | `GET/POST /api/ads/v1/campaigns` | list and run campaigns | | `GET/POST /api/ads/v1/slots` | list and create publisher slots | diff --git a/app/(marketing)/docs/cli/page.tsx b/app/(marketing)/docs/cli/page.tsx index 584a306..7409a8f 100644 --- a/app/(marketing)/docs/cli/page.tsx +++ b/app/(marketing)/docs/cli/page.tsx @@ -75,6 +75,75 @@ crawlproof dashboard --sites=example.com,blog.example.com`} crawlproof dashboard --json | jq '.sites[] | {site, visitors}'`} +
+

One property at a time

+

+ On Traffic, ↑/ + ↓ pick a site and{" "} + Enter — or a click on the row — opens + it. That screen is only that domain: pageviews, visits, humans against + bots, AI referrals and where they arrived from, next to the burn + prorated onto it, its ad earnings and spend, and the commission when + there is exactly one merchant business to attribute it to.{" "} + Esc comes back. +

+ + {`crawlproof dashboard --sort=score +crawlproof dashboard --json | jq '.sites[] | {site, score: .score.score}'`} + +
+ +
+

The risk-to-viral score

+

+ Every property is scored out of 100, shown as a column on the Traffic + list and taken apart on the domain screen. It is arithmetic over + numbers already on the screen, not a model, so a high score can always + be traced to the figure that caused it. +

+ 
{`score = 100 × viral × (1 − risk/2)
+
+viral = momentum .40 + discovery .30 + humanity .20 + money .10
+risk  = volatility .40 + concentration .30 + bot dependence .20 + unmonetised .10`}
+
    +
  • + momentum — human visits in the recent half of the + window against the earlier half. Flat is 0.5, doubling is 1. +
    • +
  • + discovery — the share arriving through search, + social, an AI assistant, an ad or another site's link, rather + than direct. +
    • +
  • + humanity — humans over humans plus bots, read + unfiltered. A filtered read has a zero bot column by construction + and would call a crawler farm 100% human. +
    • +
  • + money — revenue per 1,000 human visits against a $2 + target. +
    • +
  • + volatility — the coefficient of variation of the + human series; scale-free, so a small site is not penalised for being + small. +
    • +
  • + concentration — the largest single arrival + channel's share. An even spread is not a risk; one channel + being everything is. +
    • +
+

+ A component with no data behind it is dropped and its weight + redistributed, never counted as a zero. A trailing{" "} + ~ marks fewer than 25 human visits + in the window, and a site whose stats call failed is not scored at + all — missing is not the same as bad. +

+
+

Traffic, as text

diff --git a/app/api/tracker/v1/stats/route.ts b/app/api/tracker/v1/stats/route.ts index d862a5a..3ffed05 100644 --- a/app/api/tracker/v1/stats/route.ts +++ b/app/api/tracker/v1/stats/route.ts @@ -18,6 +18,11 @@ import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; +/** Anything but an affirmative is off, so a typo cannot buy an extra query. */ +export function parseDetail(raw: string | null): boolean { + return raw === "1" || raw === "true" || raw === "yes"; +} + export async function GET(req: NextRequest) { const auth = await authenticateBearer(req); if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); @@ -36,7 +41,13 @@ export async function GET(req: NextRequest) { const resolved = await resolveProject(sb, auth.userId, sp.get("site")); if (!resolved.ok) return NextResponse.json({ error: resolved.error }, { status: resolved.status }); + // `detail=1` adds the series and the unfiltered human / bot mix, which is + // what the dashboard's per-domain screen and its risk-to-viral score are + // built from. Off by default: `crawlproof stats` prints neither, and a + // fleet-wide fan-out should not pay for a panel nobody renders. + const detail = parseDetail(sp.get("detail")); + const range = trackerRange(sp.get("range")); - const stats = await projectStats(sb, resolved.project, range, whoToKind(who), who); + const stats = await projectStats(sb, resolved.project, range, whoToKind(who), who, detail); return NextResponse.json(stats); } diff --git a/cli/dashboard.ts b/cli/dashboard.ts index 54a1b64..7d76d5e 100644 --- a/cli/dashboard.ts +++ b/cli/dashboard.ts @@ -9,12 +9,18 @@ import type { Container, RenderArgs, Theme } from "@profullstack/hqtui"; -import { collectDashboard, type CoinPayAuth, type DashboardSnapshot } from "../lib/dashboard/collect"; +import { collectDashboard, type CoinPayAuth, type DashboardSnapshot, type SiteStats } from "../lib/dashboard/collect"; import { AD_TARGET_CTR, AD_TARGET_IMPRESSIONS, adTargets } from "../lib/dashboard/roi"; +import { buildSiteDetail, type SiteDetail } from "../lib/dashboard/site"; +import type { Component } from "../lib/dashboard/score"; export const TABS = ["ROI", "Traffic", "Ads", "Money", "Spend"] as const; export const RANGES = ["1h", "4h", "1d", "1w", "1m"] as const; +/** How the Traffic list is ordered. `s` cycles it. */ +export const SORTS = ["score", "visitors", "pageviews"] as const; +export type Sort = (typeof SORTS)[number]; + /** Which tracker range pairs with which CoinPay window. */ export const FINANCE_DAYS: Record = { "1h": 7, @@ -75,7 +81,7 @@ const clock = () => new Date().toLocaleTimeString("en-US", { hour12: false }); type Pane = { selected: number; offset: number; total: number }; -type State = { +export type State = { tab: number; range: string; who: string; @@ -88,6 +94,15 @@ type State = { panes: Record; targetImpressions: number; targetCtr: number; + /** + * The property the Traffic screen has drilled into, by name. + * + * By name rather than by index, because the list reorders on every refresh + * and on every sort — an index would silently open a different domain the + * moment anything moved. + */ + domain: string | null; + sort: Sort; }; function pane(state: State, name: string, total: number): Pane { @@ -295,47 +310,123 @@ function roiScreen(ui: Container, state: State, theme: Theme): void { }); } +/** + * The properties, in whatever order was asked for. + * + * Sites that did not answer sort last whatever the key, because a site with no + * numbers is not a site with low numbers; and a null score sorts below a real + * one rather than above it, which is what `?? 0` would have done. + */ +export function sortSites(sites: SiteStats[], sort: Sort): SiteStats[] { + const key = (row: SiteStats): number => { + if (sort === "visitors") return num(row.visitors); + if (sort === "pageviews") return num(row.pageviews); + return row.score?.score ?? -1; + }; + return [...sites].sort((a, b) => { + if (Boolean(a.error) !== Boolean(b.error)) return a.error ? 1 : -1; + return key(b) - key(a) || a.site.localeCompare(b.site); + }); +} + +type ThemeColor = Theme["success"]; + +/** Score bands, so a glance at the column means something without reading it. */ +export function scoreColor(theme: Theme, score: number | null | undefined): ThemeColor | undefined { + if (score === null || score === undefined) return theme.muted; + if (score >= 60) return theme.success; + if (score >= 35) return theme.primary; + if (score >= 15) return theme.warning; + return theme.muted; +} + +export function scoreText(row: SiteStats): string { + const value = row.score?.score; + if (value === null || value === undefined) return "—"; + // A trailing ~ marks a score from too small a sample to lean on. The number + // is still shown: hiding it would only move the guess into someone's head. + return `${value.toFixed(0)}${row.score?.provisional ? "~" : ""}`; +} + function trafficScreen(ui: Container, state: State, theme: Theme): void { const s = state.snapshot as DashboardSnapshot; - const rows = s.sites; + const rows = sortSites(s.sites, state.sort); ui.grid({ columns: ["3fr", "2fr"], gap: 1 }, (grid) => { grid.panel( { title: `Sites · ${state.range} · ${state.who}`, - subtitle: `${s.roi.attention.sitesReporting} of ${rows.length} reporting`, - footer: "j/k scroll", + subtitle: `${s.roi.attention.sitesReporting} of ${rows.length} reporting · by ${state.sort}`, + footer: "↑/↓ select · Enter opens · s sorts", }, (p) => { const view = pane(state, "sites", rows.length); - p.table({ + /** Absolute row indexes the table drew this frame, in screen order. */ + const drawn: number[] = []; + type SiteRow = { + site: string; + score: string; + scoreValue: number | null; + visitors: string; + pageviews: string; + cost: string; + note: string; + }; + p.table({ columns: [ { key: "site", title: "Site", - width: 28, + width: 26, // A site that did not answer is coloured, not silently ordinary. - color: (row: { note?: string }) => (row.note ? theme.danger : undefined), + color: (row: SiteRow) => (row.note ? theme.danger : undefined), + }, + { + key: "score", + title: "Score", + align: "right", + width: 6, + color: (row: SiteRow) => scoreColor(theme, row.scoreValue), }, { key: "visitors", title: "Visitors", align: "right", width: 10 }, { key: "pageviews", title: "Views", align: "right", width: 9 }, { key: "cost", title: "Cost", align: "right", width: 10 }, - { key: "note", title: "", width: 16, color: theme.danger }, + { key: "note", title: "", width: 14, color: theme.danger }, ], rows: rows.map((row) => { const share = s.roi.attention.visitors > 0 ? row.visitors / s.roi.attention.visitors : 0; return { site: row.site, + score: row.error ? "—" : scoreText(row), + scoreValue: row.error ? null : (row.score?.score ?? null), visitors: row.error ? "—" : count(row.visitors), pageviews: row.error ? "—" : count(row.pageviews), cost: row.error ? "—" : money(s.roi.cost.windowUsd * share, { cents: true }), - note: row.error ? row.error.slice(0, 16) : "", + note: row.error ? row.error.slice(0, 14) : "", }; }), offset: view.offset, selected: view.selected, + // The table scrolls itself to keep the selection visible, which means + // it — not this — knows where the window actually starts. `onRow` + // reports that back, both so the stored offset stays truthful and so + // a click can be mapped to a row rather than to an assumption. + followSelection: true, scrollbar: true, + onRow: (_row: SiteRow, index: number) => { + if (!drawn.length) view.offset = index; + drawn.push(index); + }, onScroll: (delta: number) => scrollPane(view, delta, 3), + // A click is a selection and an open, the way a click on a row in any + // other list is. Row 0 is the first body row; the header only focuses. + onSelectRow: (visibleRow: number) => { + const index = drawn[visibleRow]; + const row = index === undefined ? undefined : rows[index]; + if (!row || index === undefined) return; + view.selected = index; + state.domain = row.site; + }, }); }, ); @@ -375,6 +466,214 @@ function trafficScreen(ui: Container, state: State, theme: Theme): void { }); } +/** The site the Traffic screen has opened, if it is still in the snapshot. */ +export function selectedSite(state: State): SiteStats | null { + if (!state.domain || !state.snapshot) return null; + return state.snapshot.sites.find((s) => s.site === state.domain) ?? null; +} + +/** Rebuild one property's whole picture from the snapshot already in hand. */ +export function detailFor(state: State): SiteDetail | null { + const site = selectedSite(state); + const s = state.snapshot; + if (!site || !s) return null; + return buildSiteDetail({ + site, + roi: s.roi, + ads: s.ads, + finance: s.finance, + window: { range: s.window.range, who: s.window.who, financeDays: s.window.financeDays }, + }); +} + +type ComponentRow = { part: string; value: string; why: string; color: ThemeColor | undefined }; + +const componentRows = (components: Component[], theme: Theme): ComponentRow[] => + components.map((c) => ({ + part: `${c.label} ·${(c.weight * 100).toFixed(0)}`, + value: c.value === null ? "—" : pct(c.value, 0), + why: c.detail, + color: c.value === null ? theme.muted : undefined, + })); + +/** + * One property, on its own: what arrived, what it cost and earned, and why it + * scores what it scores. + * + * Every figure comes from the snapshot the list was drawn from, so opening a + * domain cannot show a number the row behind it disagreed with, and Esc goes + * back to exactly the list that was there. + */ +function domainScreen(ui: Container, state: State, theme: Theme): void { + const detail = detailFor(state); + if (!detail) { + ui.panel({ title: state.domain ?? "Site" }, (p) => { + p.text("That site is not in the current snapshot.", { fg: theme.warning }); + p.text("Esc goes back to the list.", { fg: theme.muted }); + }); + return; + } + + const t = detail.traffic; + const m = detail.money; + const score = detail.score; + + ui.grid({ columns: ["1fr", "1fr", "1fr"], rows: [12, "1fr"], gap: 1 }, (grid) => { + grid.panel( + { + title: detail.site, + subtitle: `${detail.window.range} · ${detail.window.who}`, + titleColor: theme.title, + }, + (p) => { + if (detail.error) { + p.text(detail.error, { fg: theme.danger }); + p.text("Its numbers are missing, not zero.", { fg: theme.muted }); + return; + } + p.keyValues( + [ + { label: "Pageviews", value: count(t.pageviews), color: theme.primary }, + { label: "Visits", value: count(t.visitors) }, + { label: "Humans", value: t.mixKnown ? count(t.humans) : "—", color: theme.success }, + { label: "Bots", value: t.mixKnown ? count(t.bots) : "—", color: theme.warning }, + { + label: "Human share", + value: pct(t.humanShare, 0), + color: (t.humanShare ?? 1) < 0.5 ? theme.warning : theme.success, + }, + { label: "AI referrals", value: count(t.aiReferrals) }, + { label: "", value: "" }, + { label: "Share of views", value: pct(t.viewShare, 1) }, + { label: "Share of visits", value: pct(t.visitShare, 1) }, + ], + { labelWidth: 15 }, + ); + const line = t.series.map((point) => num(point.humans)); + if (line.length > 1) { + p.sparkline({ values: line, color: theme.success, label: "humans", text: count(t.humans) }); + } + }, + ); + + // Short subtitles on purpose: hqtui draws the title and the subtitle in the + // same border row and the subtitle wins, so a long one costs the panel its + // own name at a narrow width. + grid.panel({ title: "Money", subtitle: `${detail.window.range} · ${detail.window.financeDays}d bank` }, (p) => { + p.keyValues( + [ + { + label: "Cost · by views", + value: m.costByViewsUsd === null ? "—" : money(m.costByViewsUsd, { cents: true }), + color: theme.danger, + }, + { + label: "Cost · by visits", + value: m.costByVisitsUsd === null ? "—" : money(m.costByVisitsUsd, { cents: true }), + color: theme.muted, + }, + { + label: "Revenue", + value: m.revenueUsd === null ? "—" : money(m.revenueUsd, { cents: true }), + color: theme.success, + }, + { + label: "Net", + value: m.netUsd === null ? "—" : money(m.netUsd, { cents: true }), + color: m.netUsd === null ? theme.muted : signed(theme, m.netUsd), + }, + { + label: "Per 1k humans", + value: m.rpmUsd === null ? "—" : money(m.rpmUsd, { cents: true }), + }, + { label: "", value: "" }, + // Internal by construction: one account owns the slot and the + // campaign, so this is the same dollar in two pockets. + { label: "Ad earned (int.)", value: money(m.adEarnedUsd, { cents: true }), color: theme.muted }, + { label: "Ad spent (int.)", value: money(m.adSpentUsd, { cents: true }), color: theme.muted }, + { label: "Impressions", value: count(m.adImpressions), color: theme.muted }, + ], + { labelWidth: 17 }, + ); + }); + + grid.panel( + { + title: "Risk-to-viral", + subtitle: score.provisional ? "provisional" : `${pct(score.coverage, 0)} scored`, + subtitleColor: score.provisional ? theme.warning : theme.muted, + }, + (p) => { + p.text(score.score === null ? " —" : ` ${score.score.toFixed(0)}`, { + bold: true, + fg: scoreColor(theme, score.score), + }); + p.meters( + [ + { label: "viral", value: score.viral, max: 1, text: pct(score.viral, 0) }, + { label: "risk", value: score.risk, max: 1, text: pct(score.risk, 0) }, + ], + { labelWidth: 7, valueWidth: 6 }, + ); + p.text("100 × viral × (1 − risk/2)", { fg: theme.muted }); + for (const note of score.notes.slice(0, 2)) p.text(`· ${note}`, { fg: theme.warning, wrap: true }); + }, + ); + + grid.panel({ title: "Why it scores that", subtitle: "weights", colSpan: 2 }, (p) => { + p.table({ + columns: [ + { key: "part", title: "Viral", width: 23 }, + { key: "value", title: "", align: "right", width: 6 }, + { key: "why", title: "", width: 41, color: theme.muted }, + ], + rows: componentRows(score.viralComponents, theme), + rowColor: (row: ComponentRow) => row.color, + }); + p.table({ + columns: [ + { key: "part", title: "Risk", width: 23 }, + { key: "value", title: "", align: "right", width: 6 }, + { key: "why", title: "", width: 41, color: theme.muted }, + ], + rows: componentRows(score.riskComponents, theme), + rowColor: (row: ComponentRow) => row.color, + }); + for (const gap of detail.gaps.slice(0, 3)) p.text(`· ${gap}`, { fg: theme.muted, wrap: true }); + }); + + grid.cell({ gap: 1 }, (col) => { + col.panel({ title: "Where they came from" }, (p) => { + if (!t.sources.length) { + p.text("Nobody arrived in this window.", { fg: theme.muted }); + return; + } + const max = Math.max(1, ...t.sources.map((x) => num(x.value))); + p.meters( + t.sources.slice(0, 6).map((x) => ({ + label: x.label.slice(0, 20), + value: num(x.value), + max, + text: count(x.value), + })), + { labelWidth: 21, valueWidth: 7 }, + ); + }); + + col.panel({ title: "Most-read pages" }, (p) => { + if (!t.pages.length) { + p.text("No pages read in this window.", { fg: theme.muted }); + return; + } + p.keyValues( + t.pages.slice(0, 6).map((x) => ({ label: x.label.slice(0, 26), value: count(x.value) })), + { labelWidth: 27 }, + ); + }); + }); + }); +} + function adsScreen(ui: Container, state: State, theme: Theme): void { const s = state.snapshot as DashboardSnapshot; const ads = s.ads; @@ -674,6 +973,193 @@ function spendScreen(ui: Container, state: State, theme: Theme): void { const SCREENS = [roiScreen, trafficScreen, adsScreen, moneyScreen, spendScreen]; +/** + * Draw whichever screen the state is on. + * + * One function rather than an index into SCREENS at the call site, because the + * Traffic tab has two screens — the list and one property — and the choice + * between them is state, not a tab. Exported so the render tests draw exactly + * what the app draws. + */ +export function renderBody(ui: Container, state: State, theme: Theme): void { + if (!state.snapshot) { + ui.panel({ title: "Spend & ROI" }, (p) => { + if (state.error) { + p.text(`Could not load: ${state.error}`, { fg: theme.danger }); + p.text("Press r to retry, q to quit.", { fg: theme.muted }); + } else { + p.text("Reading the fleet…", { fg: theme.muted }); + p.text("One tracker call per site, plus ad earnings and CoinPay.", { fg: theme.muted }); + } + }); + return; + } + if (state.tab === 1 && state.domain) { + domainScreen(ui, state, theme); + return; + } + (SCREENS[state.tab] ?? roiScreen)(ui, state, theme); +} + +/** A fresh state, for `runDashboard` and for the render tests alike. */ +export function initialState(overrides: Partial = {}): State { + return { + tab: 0, + range: "1d", + who: "humans", + snapshot: null, + loading: false, + lastRefresh: null, + error: null, + paused: false, + showHelp: false, + panes: {}, + targetImpressions: AD_TARGET_IMPRESSIONS, + targetCtr: AD_TARGET_CTR, + domain: null, + sort: "score", + ...overrides, + }; +} + +/** Move the highlighted row, letting the table work out the scroll. */ +function moveSelection(p: Pane, delta: number): void { + const max = Math.max(0, p.total - 1); + p.selected = Math.max(0, Math.min(p.selected + delta, max)); +} + +export type KeyLike = { name: string; shift?: boolean }; + +/** + * Every key the dashboard answers to, as a pure function of the state. + * + * Pulled out of the app so the navigation that matters — opening a domain, + * coming back from it, re-sorting without losing your place — can be tested + * without a terminal. Returns true when something changed and the frame is + * worth redrawing. + */ +export function handleKey(state: State, event: KeyLike, actions: { refresh: () => void }): boolean { + if (state.showHelp) { + state.showHelp = false; + return true; + } + + const onTraffic = state.tab === 1; + const sites = state.snapshot ? sortSites(state.snapshot.sites, state.sort) : []; + const view = pane(state, TAB_PANE[state.tab] as string, state.panes[TAB_PANE[state.tab] as string]?.total ?? 0); + + // A number is always the top-level screen it names, so 2 is the way back to + // the list from a domain as well as the way to the Traffic tab from anywhere. + const digit = Number(event.name); + if (Number.isInteger(digit) && event.name.length === 1 && digit >= 1 && digit <= TABS.length) { + state.tab = digit - 1; + state.domain = null; + return true; + } + + switch (event.name) { + case "escape": + case "backspace": + if (!state.domain) return false; + state.domain = null; + return true; + + case "enter": + case "return": + case "right": { + // → opens a domain from the list, and otherwise keeps its old job of + // moving to the next screen. + if (onTraffic && !state.domain && sites.length) { + const row = sites[Math.min(view.selected, sites.length - 1)]; + if (row) { + state.domain = row.site; + return true; + } + } + if (event.name !== "right") return false; + state.tab = event.shift ? (state.tab + TABS.length - 1) % TABS.length : (state.tab + 1) % TABS.length; + return true; + } + + case "tab": + case "l": + state.tab = event.shift ? (state.tab + TABS.length - 1) % TABS.length : (state.tab + 1) % TABS.length; + return true; + + case "left": + case "h": + // ← is the way back out of a domain too, since that is where it came from. + if (state.domain) { + state.domain = null; + return true; + } + state.tab = (state.tab + TABS.length - 1) % TABS.length; + return true; + + case "s": { + // Re-sorting keeps the highlight on the same property rather than on the + // same row number, which is the only version of this that is not annoying. + const held = sites[view.selected]?.site ?? null; + state.sort = SORTS[(SORTS.indexOf(state.sort) + 1) % SORTS.length] as Sort; + if (held && state.snapshot) { + const next = sortSites(state.snapshot.sites, state.sort).findIndex((row) => row.site === held); + if (next >= 0) view.selected = next; + } + return true; + } + + case "r": + case "f5": + actions.refresh(); + return true; + + case "w": + state.range = RANGES[(RANGES.indexOf(state.range as never) + 1) % RANGES.length] as string; + actions.refresh(); + return true; + + case "b": + state.who = state.who === "humans" ? "all" : state.who === "all" ? "bots" : "humans"; + actions.refresh(); + return true; + + case "p": + case "space": + state.paused = !state.paused; + return true; + + case "?": + case "f1": + state.showHelp = true; + return true; + + case "up": + case "k": + if (onTraffic && !state.domain) moveSelection(view, -1); + else scrollPane(view, -1); + return true; + + case "down": + case "j": + if (onTraffic && !state.domain) moveSelection(view, 1); + else scrollPane(view, 1); + return true; + + case "pageup": + if (onTraffic && !state.domain) moveSelection(view, -10); + else scrollPane(view, -1, 10); + return true; + + case "pagedown": + if (onTraffic && !state.domain) moveSelection(view, 10); + else scrollPane(view, 1, 10); + return true; + + default: + return false; + } +} + // ── app ── async function loadHqtui(): Promise { @@ -702,6 +1188,8 @@ export type DashboardOptions = { /** Where the ad network is trying to get to; see lib/dashboard/roi.ts. */ targetImpressions?: number; targetCtr?: number; + /** Initial order of the Traffic list: score, visitors or pageviews. */ + sort?: string; }; export async function runDashboard(opts: DashboardOptions): Promise { @@ -712,20 +1200,13 @@ export async function runDashboard(opts: DashboardOptions): Promise { quitKeys: ["ctrl+c", "q"], }); - const state: State = { - tab: 0, + const state: State = initialState({ range: opts.range && RANGES.includes(opts.range as never) ? opts.range : "1d", who: opts.who ?? "humans", - snapshot: null, - loading: false, - lastRefresh: null, - error: null, - paused: false, - showHelp: false, - panes: {}, targetImpressions: opts.targetImpressions ?? AD_TARGET_IMPRESSIONS, targetCtr: opts.targetCtr ?? AD_TARGET_CTR, - }; + ...(opts.sort && SORTS.includes(opts.sort as never) ? { sort: opts.sort as Sort } : {}), + }); let refreshing = false; @@ -765,67 +1246,8 @@ export async function runDashboard(opts: DashboardOptions): Promise { const tick = setInterval(() => app.invalidate(), 1000); tick.unref?.(); - app.on("key", (event: { name: string; shift?: boolean }) => { - if (state.showHelp) { - state.showHelp = false; - app.invalidate(); - return; - } - const digit = Number(event.name); - if (Number.isInteger(digit) && event.name.length === 1 && digit >= 1 && digit <= TABS.length) { - state.tab = digit - 1; - app.invalidate(); - return; - } - const view = pane(state, TAB_PANE[state.tab] as string, state.panes[TAB_PANE[state.tab] as string]?.total ?? 0); - switch (event.name) { - case "tab": - case "right": - case "l": - state.tab = event.shift ? (state.tab + TABS.length - 1) % TABS.length : (state.tab + 1) % TABS.length; - break; - case "left": - case "h": - state.tab = (state.tab + TABS.length - 1) % TABS.length; - break; - case "r": - case "f5": - void refresh(); - break; - case "w": - state.range = RANGES[(RANGES.indexOf(state.range as never) + 1) % RANGES.length] as string; - void refresh(); - break; - case "b": - state.who = state.who === "humans" ? "all" : state.who === "all" ? "bots" : "humans"; - void refresh(); - break; - case "p": - case "space": - state.paused = !state.paused; - break; - case "?": - case "f1": - state.showHelp = true; - break; - case "up": - case "k": - scrollPane(view, -1); - break; - case "down": - case "j": - scrollPane(view, 1); - break; - case "pageup": - scrollPane(view, -1, 10); - break; - case "pagedown": - scrollPane(view, 1, 10); - break; - default: - return; - } - app.invalidate(); + app.on("key", (event: KeyLike) => { + if (handleKey(state, event, { refresh })) app.invalidate(); }); app.render(({ ui, theme, height }: RenderArgs) => { @@ -850,27 +1272,17 @@ export async function runDashboard(opts: DashboardOptions): Promise { }); ui.spacer(1); - ui.column({ size: height - 4 }, (body) => { - if (!state.snapshot) { - body.panel({ title: "Spend & ROI" }, (p) => { - if (state.error) { - p.text(`Could not load: ${state.error}`, { fg: theme.danger }); - p.text("Press r to retry, q to quit.", { fg: theme.muted }); - } else { - p.text("Reading the fleet…", { fg: theme.muted }); - p.text("One tracker call per site, plus ad earnings and CoinPay.", { fg: theme.muted }); - } - }); - return; - } - (SCREENS[state.tab] ?? roiScreen)(body, state, theme); - }); + ui.column({ size: height - 4 }, (body) => renderBody(body, state, theme)); ui.spacer(1); const errorCount = state.snapshot ? Object.keys(state.snapshot.errors).length : 0; + const onList = state.tab === 1 && !state.domain; ui.statusBar({ items: [ { key: "1-5", label: "Screen" }, + ...(onList ? [{ key: "↵", label: "Open site" }] : []), + ...(state.domain ? [{ key: "esc", label: "Back", active: true }] : []), + ...(onList ? [{ key: "s", label: `Sort ${state.sort}` }] : []), { key: "r", label: "Refresh" }, { key: "w", label: `Window ${state.range}` }, { key: "b", label: state.who }, @@ -886,21 +1298,28 @@ export async function runDashboard(opts: DashboardOptions): Promise { if (state.showHelp) { ui.modal({ title: "CrawlProof — Spend & ROI", - width: 70, - height: 22, + width: 76, + height: 28, message: "1-5, Tab, ←/→ switch screens.\n" + `r refreshes now; it also refreshes every ${interval}s.\n` + "w cycles the window: 1h → 4h → 1d → 1w → 1m.\n" + "b cycles who counts: humans → all → bots.\n" + - "p pauses the timer. ↑/↓ j/k, PgUp/PgDn scroll a table.\n\n" + + "p pauses the timer. ↑/↓ j/k, PgUp/PgDn move in a table.\n\n" + + "On Traffic: ↑/↓ pick a site, Enter or a click opens it,\n" + + " Esc / ← / 2 comes back, s cycles the order:\n" + + " score → visitors → pageviews.\n\n" + + "Risk-to-viral scores a property out of 100:\n" + + " score = 100 × viral × (1 − risk/2)\n" + + " viral = momentum .40 + discovery .30 + humanity .20 + money .10\n" + + " risk = volatility .40 + concentration .30 + bots .20 + unmonetised .10\n" + + " A component with no data is dropped, not counted as zero;\n" + + " ~ marks too small a sample. The domain screen shows every part.\n\n" + "Cost is business-scope bank spend as a monthly rate, so it\n" + " does not move when you change the traffic window.\n" + "Revenue is CoinPay commission only. Ad spend and ad earnings\n" + " are the same account on both sides of our own network, so\n" + - " they are reported under Internal and counted as neither.\n" + - "Cost each = the monthly burn prorated onto the window,\n" + - " divided by the visitors who arrived in it.\n\n" + + " they are reported under Internal and counted as neither.\n\n" + "Press any key to close.", buttons: [{ label: "Close", focused: true }], }); diff --git a/cli/index.ts b/cli/index.ts index 1f0afc1..a98c31b 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -504,6 +504,7 @@ async function cmdDashboard(args: Args): Promise { concurrency: Number(args.flags.concurrency) || 8, coinpay, only, + sort: args.flags.sort as string | undefined, theme: args.flags.theme as string | undefined, }); return 0; @@ -573,13 +574,24 @@ COMMANDS project name; with one project it can be left out. Needs an API token. dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--interval=60] - [--sites=a.com,b.com] [--concurrency=8] [--no-coinpay] [--json] + [--sites=a.com,b.com] [--sort=score|visitors|pageviews] + [--concurrency=8] [--no-coinpay] [--json] A live terminal dashboard of what the fleet costs and what it returns: traffic across every site you own, ad delivery, and — when a CoinPay merchant session is on the box — the bank feed behind it. Five screens: ROI, Traffic, Ads, Money, Spend. Needs an API token and a terminal; --json prints the same snapshot for a script. Aliases: roi, tui. + On Traffic, ↑/↓ pick a property and Enter (or a click) opens it: that + domain's traffic and money on their own, with its risk-to-viral score + broken into the parts it was built from. Esc / ← / 2 comes back, s + cycles the order. The score is + 100 × viral × (1 − risk/2), viral = momentum .40 + discovery .30 + + humanity .20 + money .10, risk = volatility .40 + concentration .30 + + bot dependence .20 + unmonetised .10. + A component with no data is dropped, not counted as zero; ~ marks a + sample under 25 human visits. + help Print this message. @@ -607,6 +619,7 @@ EXAMPLES CRAWLPROOF_TOKEN=crp_... crawlproof slots create nichedb.dev CRAWLPROOF_TOKEN=crp_... crawlproof dashboard --range=1w CRAWLPROOF_TOKEN=crp_... crawlproof dashboard --json | jq .roi.derived + CRAWLPROOF_TOKEN=crp_... crawlproof dashboard --json | jq '.sites[] | {site, score: .score.score}' `); } diff --git a/lib/ads/earnings-data.ts b/lib/ads/earnings-data.ts index e4c8f09..bad2b74 100644 --- a/lib/ads/earnings-data.ts +++ b/lib/ads/earnings-data.ts @@ -27,6 +27,12 @@ export type EarningsCampaignRow = { id: string; name: string; status: string; + /** + * Where the campaign points. Carried so a client can attribute spend to one + * of its own properties: a campaign is only ever "for" a site by way of the + * page it sends people to, and nothing else in this model names a domain. + */ + url: string | null; impressions: number; clicks: number; spentCents: number; @@ -35,6 +41,8 @@ export type EarningsCampaignRow = { export type EarningsSlotRow = { id: string; name: string; + /** The project the slot sits on, so earnings can be read per property. */ + projectId: string; status: string; impressions: number; clicks: number; @@ -103,6 +111,7 @@ type CampaignRow = { id: string; name: string; status: string; + destination_url: string | null; total_spent_cents: number | null; spend_today_cents: number | null; spend_date: string | null; @@ -138,7 +147,7 @@ export async function loadEarnings( ] = await Promise.all([ supabase .from("ad_campaigns") - .select("id, name, status, total_spent_cents, spend_today_cents, spend_date") + .select("id, name, status, destination_url, total_spent_cents, spend_today_cents, spend_date") .eq("owner_id", userId), // Not ad_campaign_stats / ad_slot_stats: those views are lifetime and count // only tier 'paid', so on a network running entirely on free backfill they @@ -179,6 +188,7 @@ export async function loadEarnings( id: c.id, name: c.name, status: c.status, + url: c.destination_url ?? null, impressions: deliveredImpressions(s), clicks: deliveredClicks(s), spentCents: s.spentCents, @@ -190,6 +200,7 @@ export async function loadEarnings( return { id: sl.id, name: projectsById.get(sl.project_id)?.name ?? "Site", + projectId: sl.project_id, status: sl.status, impressions: deliveredImpressions(s), clicks: deliveredClicks(s), diff --git a/lib/dashboard/collect.ts b/lib/dashboard/collect.ts index 30bc466..9e0eb7e 100644 --- a/lib/dashboard/collect.ts +++ b/lib/dashboard/collect.ts @@ -20,6 +20,8 @@ import { type RoiModel, type SiteTraffic, } from "./roi"; +import type { ScoreModel } from "./score"; +import { buildSiteDetail, type SiteMix, type SitePoint } from "./site"; export type ListItem = { label: string; value: number }; @@ -29,6 +31,16 @@ export type SiteStats = SiteTraffic & { sources: ListItem[]; referrers: ListItem[]; pages: ListItem[]; + /** The shape over the window, for the domain screen and the score. */ + series?: SitePoint[]; + /** Humans against bots, unfiltered. Absent when the API did not answer it. */ + mix?: SiteMix; + /** + * The risk-to-viral score for this property; see lib/dashboard/score.ts. + * Attached here so the Traffic list can rank by it without every screen + * recomputing it, and so `--json` carries it for a script. + */ + score?: ScoreModel; }; export type DashboardSnapshot = { @@ -119,13 +131,19 @@ async function statsForSite( range: string, who: string, ): Promise { - const url = `${baseUrl}/api/tracker/v1/stats?site=${encodeURIComponent(site.id)}&range=${encodeURIComponent(range)}&who=${encodeURIComponent(who)}`; + // `detail=1` asks for the series and the unfiltered human / bot mix. Both are + // per-domain questions — the fleet screens need neither — and the mix is the + // only honest source for "how much of this is a crawler", because a filtered + // series has a zero bot column by construction. + const url = `${baseUrl}/api/tracker/v1/stats?site=${encodeURIComponent(site.id)}&range=${encodeURIComponent(range)}&who=${encodeURIComponent(who)}&detail=1`; try { const body = await fetchJson<{ totals?: { visitors?: number; pageviews?: number }; sources?: ListItem[]; referrers?: ListItem[]; pages?: ListItem[]; + series?: SitePoint[]; + mix?: SiteMix; }>(url, token); return { site: site.name, @@ -136,6 +154,8 @@ async function statsForSite( sources: body.sources ?? [], referrers: body.referrers ?? [], pages: body.pages ?? [], + ...(body.series ? { series: body.series } : {}), + ...(body.mix ? { mix: body.mix } : {}), }; } catch (err) { return { @@ -234,6 +254,14 @@ export async function collectDashboard(opts: CollectOptions): Promise; + /** Per-campaign spend. `url` is the only field that names a domain. */ + campaigns?: Array<{ + id?: string; + name?: string; + status?: string; + url?: string | null; + impressions?: number; + clicks?: number; + spentCents?: number; + }>; totals?: { spentCents?: number; earnedCents?: number; @@ -76,6 +100,11 @@ export type AdsInput = { /** The subset of the CoinPay finance snapshot this module reads. */ export type FinanceInput = { windowDays?: number; + /** + * The merchant's businesses. Read by lib/dashboard/site.ts, which can only + * attribute commission to a domain when there is exactly one of them. + */ + businesses?: Array<{ id?: string; name?: string }>; /** * The headline earnings figures, which are **lifetime and not windowed**. * diff --git a/lib/dashboard/score.ts b/lib/dashboard/score.ts new file mode 100644 index 0000000..82076de --- /dev/null +++ b/lib/dashboard/score.ts @@ -0,0 +1,390 @@ +// The risk-to-viral score: which property is worth another week of attention. +// +// WHY a score at all. The Traffic screen ranks sites by visitors, and visitors +// is the one number on this fleet that lies — most of it is crawler traffic +// (lib/tracker/humans.ts), and on a site with a machine-readable endpoint a +// "visitor" is any hit that was not classified as a crawler, which runs orders +// of magnitude above the pages anyone read. So the busiest row is routinely the +// least interesting one. This ranks by the thing that is actually being asked: +// is this property going anywhere, and how fragile is the answer. +// +// It is deliberately arithmetic rather than a model. Every component is one +// division over numbers already on the screen, every weight is written down +// here and rendered on the domain screen, and a component with no data is +// dropped and its weight redistributed rather than counted as a zero. A score +// nobody can take apart is a score nobody should act on. +// +// ── THE FORMULA ──────────────────────────────────────────────────────────── +// +// Two halves, each a weighted mean of components in 0..1. +// +// VIRAL — the upside +// momentum 0.40 human visits in the recent half of the window against +// the earlier half: g = (recent - prior) / max(prior, 1), +// scored 0.5 + g/2, so flat is 0.5, doubling is 1.0 and +// losing everything is 0. +// discovery 0.30 share of human arrivals from a channel a stranger can +// come through — search, social, AI referral, ad, another +// site's link — rather than direct or our own referral. +// Reach that compounds, as opposed to reach we already had. +// humanity 0.20 humans / (humans + bots). Needs the unfiltered mix; when +// the window was asked for one side only it is unknown and +// this component is dropped rather than assumed. +// money 0.10 revenue per 1,000 human visits against TARGET_RPM_USD. +// Small on purpose: nothing on this fleet is monetised yet +// and weighting it heavily would score every property 0. +// +// RISK — the fragility, which is what "risk-to-viral" names +// volatility 0.40 coefficient of variation of the human series over +// the window, against CV_CEILING. +// concentration 0.30 the largest single arrival channel's share, rescaled +// from CONCENTRATION_FLOOR..1 onto 0..1: an even split +// across channels is not a risk, one channel being +// everything is the whole risk. +// botDependence 0.20 1 - humanity. +// unmonetised 0.10 1 - money. +// +// score = 100 × viral × (1 − risk / 2) +// +// So risk can halve a property's score and never more, and a property with no +// upside scores 0 however safe it is — which is the right shape for a question +// about where to spend the next week. +// +// Every input is coerced, every denominator is guarded, and nothing here can +// return NaN or Infinity: see `clamp01` and the `safeDiv` calls. + +/** Revenue per 1,000 human visits that counts as fully monetised. */ +export const TARGET_RPM_USD = 2; + +/** Coefficient of variation at which the volatility component saturates. */ +export const CV_CEILING = 1.5; + +/** Below this share, one dominant channel is not yet counted as a risk. */ +export const CONCENTRATION_FLOOR = 0.5; + +/** Human visits below which a score is labelled provisional rather than read. */ +export const MIN_SAMPLE_HUMANS = 25; + +export const VIRAL_WEIGHTS = { + momentum: 0.4, + discovery: 0.3, + humanity: 0.2, + money: 0.1, +} as const; + +export const RISK_WEIGHTS = { + volatility: 0.4, + concentration: 0.3, + botDependence: 0.2, + unmonetised: 0.1, +} as const; + +/** + * Arrival buckets a stranger can reach us through. + * + * `bucketLabel` in lib/tracker/categorize.ts renders the bucket as + * "Search · google", "AI · chatgpt" and so on, and the dashboard carries those + * labels rather than the raw buckets, so both spellings are matched here. + * "Direct" and a self-referral are reach we already had; they are not counted. + */ +const DISCOVERY_PREFIXES = ["search", "social", "ai", "ad", "referral", "ai_referral"]; + +const num = (v: unknown): number => { + const x = Number(v); + return Number.isFinite(x) ? x : 0; +}; + +/** + * 0..1, with NaN ordered to 0 rather than propagated. + * + * Infinity saturates rather than falling to zero: an unbounded ratio means the + * component is off the top of its scale, and reading that as "nothing here" + * would score the most volatile property as the safest one. + */ +export function clamp01(value: number): number { + if (Number.isNaN(value)) return 0; + return value < 0 ? 0 : value > 1 ? 1 : value; +} + +/** A ratio, or null when the denominator cannot carry one. */ +function safeDiv(top: number, bottom: number): number | null { + if (!Number.isFinite(top) || !Number.isFinite(bottom) || bottom <= 0) return null; + const out = top / bottom; + return Number.isFinite(out) ? out : null; +} + +export type ScoreItem = { label: string; value: number }; + +export type ScoreInput = { + /** Human visits per bucket across the window, oldest first. */ + humans: number[]; + /** Bot hits per bucket, from the unfiltered mix. Empty when it is unknown. */ + bots?: number[]; + /** Arrival channels for the window, as the dashboard already carries them. */ + sources?: ScoreItem[]; + /** Money attributable to this property over the window, in dollars. */ + revenueUsd?: number; + /** + * Human visits over the window, when a truer total than the series sum is + * known. The series is filtered to whichever side was asked for, so under + * `who=bots` its human column is zero by construction; the unfiltered mix + * knows the real figure and this is where it comes in. + */ + humansTotal?: number; + /** Bot hits over the window, from the same unfiltered read. */ + botsTotal?: number; + /** + * True when the mix of humans against bots is known for this window. It is + * not when the caller asked for one side only and nothing counted the other. + */ + mixKnown?: boolean; +}; + +export type Component = { + key: string; + label: string; + /** 0..1, or null when there was nothing to compute it from. */ + value: number | null; + weight: number; + /** The raw figure behind the component, for the detail screen. */ + detail: string; +}; + +export type ScoreModel = { + /** 0..100, or null when neither half could be computed. */ + score: number | null; + /** 0..1. */ + viral: number; + /** 0..1. */ + risk: number; + viralComponents: Component[]; + riskComponents: Component[]; + /** Human visits the score was computed over. */ + humans: number; + /** True when the sample is too small to lean on. */ + provisional: boolean; + /** Share of the weight that had data behind it, across both halves. */ + coverage: number; + /** Why a component is missing, or why the score should not be read straight. */ + notes: string[]; +}; + +const sum = (values: number[]): number => values.reduce((total, v) => total + num(v), 0); + +/** + * Growth of the recent half of the window against the earlier half. + * + * Halves rather than a fitted trend because the window is as short as thirteen + * five-minute buckets and as long as thirty days, and a fit over thirteen noisy + * buckets says more about the noise than the site. Null when there are fewer + * than four buckets, which is the point below which "recent" and "earlier" are + * the same two numbers. + */ +export function growthRate(series: number[]): number | null { + const points = (series ?? []).map(num); + if (points.length < 4) return null; + const mid = Math.floor(points.length / 2); + const prior = sum(points.slice(0, mid)); + const recent = sum(points.slice(mid)); + if (prior <= 0 && recent <= 0) return null; + // max(prior, 1) rather than a guard: from nothing to something is growth, and + // dividing by zero to say so is not. + return (recent - prior) / Math.max(prior, 1); +} + +/** + * Coefficient of variation: standard deviation over the mean. + * + * Scale-free on purpose, so a site doing 20 visits a day and one doing 20,000 + * are asked the same question — how steady is it — rather than the larger one + * always reading as the more volatile. + */ +export function coefficientOfVariation(series: number[]): number | null { + const points = (series ?? []).map(num); + if (points.length < 3) return null; + const mean = sum(points) / points.length; + if (mean <= 0) return null; + const variance = points.reduce((total, v) => total + (v - mean) ** 2, 0) / points.length; + const cv = safeDiv(Math.sqrt(variance), mean); + return cv === null ? null : cv; +} + +/** Share of arrivals from a channel a stranger can come through. */ +export function discoveryShare(sources: ScoreItem[] | undefined): number | null { + const items = (sources ?? []).filter((s) => s && typeof s.label === "string"); + const total = sum(items.map((s) => num(s.value))); + if (total <= 0) return null; + const discovered = items + .filter((s) => { + const head = s.label.split("·")[0]?.trim().toLowerCase() ?? ""; + return DISCOVERY_PREFIXES.some((p) => head === p || head.startsWith(`${p}:`) || head.startsWith(`${p} `)); + }) + .reduce((t, s) => t + num(s.value), 0); + return clamp01(discovered / total); +} + +/** The largest single channel's share of arrivals. */ +export function topSourceShare(sources: ScoreItem[] | undefined): number | null { + const values = (sources ?? []).map((s) => num(s?.value)).filter((v) => v > 0); + const total = sum(values); + if (total <= 0) return null; + return clamp01(Math.max(...values) / total); +} + +/** A weighted mean over the components that have a value, weights renormalised. */ +function weightedMean(components: Component[]): { value: number; weight: number } { + let weighted = 0; + let weight = 0; + for (const c of components) { + if (c.value === null) continue; + weighted += clamp01(c.value) * c.weight; + weight += c.weight; + } + return { value: weight > 0 ? weighted / weight : 0, weight }; +} + +const pct = (v: number | null): string => (v === null ? "no data" : `${(v * 100).toFixed(0)}%`); + +/** + * Score one property. Pure, total, and safe against every empty shape: an input + * of `{ humans: [] }` returns a null score with notes, never a NaN. + */ +export function scoreSite(input: ScoreInput): ScoreModel { + const humansSeries = (input.humans ?? []).map(num); + const botsSeries = (input.bots ?? []).map(num); + const humans = input.humansTotal === undefined ? sum(humansSeries) : num(input.humansTotal); + const bots = input.botsTotal === undefined ? sum(botsSeries) : num(input.botsTotal); + const notes: string[] = []; + + const growth = growthRate(humansSeries); + if (growth === null && humansSeries.length < 4) { + notes.push("Too few buckets in this window to measure growth; widen it with w."); + } + + const mixKnown = input.mixKnown !== false && (humans > 0 || bots > 0); + const humanity = mixKnown ? safeDiv(humans, humans + bots) : null; + if (!mixKnown) { + notes.push("Humans against bots is unknown for this window, so that component is not counted."); + } + + const discovery = discoveryShare(input.sources); + const concentrationRaw = topSourceShare(input.sources); + const cv = coefficientOfVariation(humansSeries); + + const revenueUsd = num(input.revenueUsd); + const rpm = safeDiv(revenueUsd * 1000, humans); + const money = rpm === null ? null : clamp01(rpm / TARGET_RPM_USD); + + const viralComponents: Component[] = [ + { + key: "momentum", + label: "Momentum", + value: growth === null ? null : clamp01(0.5 + growth / 2), + weight: VIRAL_WEIGHTS.momentum, + detail: + growth === null + ? "no data" + : `${growth >= 0 ? "+" : ""}${(growth * 100).toFixed(0)}% human visits, recent half vs earlier`, + }, + { + key: "discovery", + label: "Discovery", + value: discovery, + weight: VIRAL_WEIGHTS.discovery, + detail: discovery === null ? "no arrivals" : `${pct(discovery)} arrived via search, social, AI, ad or a link`, + }, + { + key: "humanity", + label: "Humanity", + value: humanity, + weight: VIRAL_WEIGHTS.humanity, + detail: + humanity === null + ? "mix unknown" + : `${humans.toLocaleString("en-US")} human of ${(humans + bots).toLocaleString("en-US")} hits`, + }, + { + key: "money", + label: "Money", + value: money, + weight: VIRAL_WEIGHTS.money, + detail: + rpm === null + ? "no human visits to divide by" + : `$${rpm.toFixed(2)} per 1k human visits (target $${TARGET_RPM_USD.toFixed(2)})`, + }, + ]; + + const concentration = + concentrationRaw === null + ? null + : clamp01((concentrationRaw - CONCENTRATION_FLOOR) / (1 - CONCENTRATION_FLOOR)); + + const riskComponents: Component[] = [ + { + key: "volatility", + label: "Volatility", + value: cv === null ? null : clamp01(cv / CV_CEILING), + weight: RISK_WEIGHTS.volatility, + detail: cv === null ? "no data" : `CV ${cv.toFixed(2)} across ${humansSeries.length} buckets`, + }, + { + key: "concentration", + label: "Channel concentration", + value: concentration, + weight: RISK_WEIGHTS.concentration, + detail: + concentrationRaw === null + ? "no arrivals" + : `biggest channel is ${pct(concentrationRaw)} of arrivals`, + }, + { + key: "botDependence", + label: "Bot dependence", + value: humanity === null ? null : clamp01(1 - humanity), + weight: RISK_WEIGHTS.botDependence, + detail: humanity === null ? "mix unknown" : `${pct(humanity === null ? null : 1 - humanity)} of hits are crawlers`, + }, + { + key: "unmonetised", + label: "Unmonetised", + value: money === null ? null : clamp01(1 - money), + weight: RISK_WEIGHTS.unmonetised, + detail: rpm === null ? "no revenue basis" : `$${revenueUsd.toFixed(2)} attributable in this window`, + }, + ]; + + const viral = weightedMean(viralComponents); + const risk = weightedMean(riskComponents); + const totalWeight = viral.weight + risk.weight; + const coverage = clamp01(totalWeight / 2); + + const provisional = humans < MIN_SAMPLE_HUMANS; + if (provisional) { + notes.push( + `Only ${humans.toLocaleString("en-US")} human visits in this window; under ${MIN_SAMPLE_HUMANS} the score is a guess with error bars.`, + ); + } + + // Nothing at all is a real answer and should read as one, rather than as the + // zero a property that was measured and found dead would get. + const score = viral.weight <= 0 && risk.weight <= 0 ? null : 100 * viral.value * (1 - risk.value / 2); + + return { + score: score === null ? null : Math.round(score * 10) / 10, + viral: viral.value, + risk: risk.value, + viralComponents, + riskComponents, + humans, + provisional, + coverage, + notes, + }; +} + +/** The one-line explanation the CLI help, the README and the screen all use. */ +export const SCORE_FORMULA = + "score = 100 × viral × (1 − risk/2), where viral is momentum .40 + discovery .30 + humanity .20 + money .10 " + + "and risk is volatility .40 + channel concentration .30 + bot dependence .20 + unmonetised .10. " + + "Components with no data are dropped and their weight redistributed."; diff --git a/lib/dashboard/site.ts b/lib/dashboard/site.ts new file mode 100644 index 0000000..cea4acf --- /dev/null +++ b/lib/dashboard/site.ts @@ -0,0 +1,305 @@ +// One property, on its own: what arrived, what it cost, what it earned. +// +// The fleet screens answer "how are we doing". This answers "is THIS domain +// worth another week", which is a different question and needs the money split +// per property rather than summed across it. +// +// Pure, and built entirely from the snapshot the dashboard already collected — +// opening a domain fires no new request, so the detail screen can never +// disagree with the list it was opened from. +// +// Three money lines, each attributable by a different join, and each says which: +// +// cost the fleet's burn, prorated onto this property by its share of +// traffic. Two of them, because the two denominators disagree by +// orders of magnitude on this fleet (see the visitors caveat in +// lib/dashboard/roi.ts) and picking one quietly would be a lie. +// ad money earnings by slot → project id, spend by campaign → +// destination host. Both exact. Both internal: this network has +// one account on both sides, so neither is revenue. +// revenue CoinPay commission, which is the only money from outside the +// fleet — and is attributable to a domain only when the merchant +// account has exactly one business and it is this one. Otherwise +// it is null and the screen says why rather than dividing the +// fleet's revenue by a number of sites. + +import type { AdsInput, FinanceInput, RoiModel } from "./roi"; +import { scoreSite, type ScoreItem, type ScoreModel } from "./score"; + +export type SitePoint = { date: string; pageviews: number; humans: number; bots: number; ai: number }; + +export type SiteMix = { humans: number; bots: number; ai: number; events: number }; + +/** What the collector carries for each property. */ +export type SiteLike = { + site: string; + id?: string; + url?: string; + visitors: number; + pageviews: number; + sources: ScoreItem[]; + referrers: ScoreItem[]; + pages: ScoreItem[]; + series?: SitePoint[]; + mix?: SiteMix; + error?: string; +}; + +export type SiteMoney = { + /** Burn prorated by this property's share of fleet pageviews. */ + costByViewsUsd: number | null; + /** The same, by its share of fleet visits. Flattering; see the module note. */ + costByVisitsUsd: number | null; + adEarnedUsd: number; + adSpentUsd: number; + adImpressions: number; + adClicks: number; + /** Money from outside the fleet, or null when it cannot be attributed here. */ + revenueUsd: number | null; + /** How `revenueUsd` was arrived at, for the screen to print. */ + revenueBasis: string; + /** Revenue per 1,000 human visits, when both halves exist. */ + rpmUsd: number | null; + /** Revenue less the cost-by-views share. Null when either half is null. */ + netUsd: number | null; +}; + +export type SiteDetail = { + site: string; + url: string | null; + error: string | null; + window: { range: string; who: string; days: number; financeDays: number }; + traffic: { + visitors: number; + pageviews: number; + humans: number; + bots: number; + aiReferrals: number; + /** humans / (humans + bots), or null when the mix is unknown. */ + humanShare: number | null; + mixKnown: boolean; + visitShare: number; + viewShare: number; + series: SitePoint[]; + sources: ScoreItem[]; + referrers: ScoreItem[]; + pages: ScoreItem[]; + }; + money: SiteMoney; + score: ScoreModel; + /** What could not be attributed to this property, named rather than guessed. */ + gaps: string[]; +}; + +const num = (v: unknown): number => { + const x = Number(v); + return Number.isFinite(x) ? x : 0; +}; + +const cents = (v: unknown): number => num(v) / 100; + +const share = (part: number, whole: number): number => (whole > 0 ? num(part) / whole : 0); + +/** + * A hostname from a URL or a bare host, lowercased and de-`www`'d. + * + * Deliberately not lib/ads/slots' `hostOf`: this module is bundled into the + * published CLI, and that one reaches through net-guard into server code. + */ +export function hostFrom(input: string | null | undefined): string | null { + const raw = String(input ?? "").trim(); + if (!raw) return null; + try { + const url = new URL(/^[a-z]+:\/\//i.test(raw) ? raw : `https://${raw}`); + return url.hostname.toLowerCase().replace(/^www\./, "") || null; + } catch { + return null; + } +} + +/** True when a site row and a URL name the same property. */ +export function sameProperty(site: SiteLike, url: string | null | undefined): boolean { + const target = hostFrom(url); + if (!target) return false; + const host = hostFrom(site.url) ?? hostFrom(site.site); + if (host && host === target) return true; + // A project named for its host but with no URL on file still matches. + return site.site.toLowerCase() === target; +} + +/** + * CoinPay commission attributable to one property. + * + * Only when the merchant account has exactly one business and it is this one. + * With several, the snapshot carries a fleet total and no per-business split — + * `getFinanceAnalytics` takes a `businessId` but the dashboard makes one call, + * not one per business — so anything else would be the fleet's revenue divided + * by a guess. + */ +export function coinpayRevenueForSite( + finance: FinanceInput | null, + roi: RoiModel, + site: SiteLike, +): { usd: number | null; basis: string } { + const businesses = (finance as { businesses?: Array<{ id?: string; name?: string }> } | null)?.businesses ?? []; + if (!finance) return { usd: null, basis: "no CoinPay session" }; + if (businesses.length !== 1) { + return { + usd: null, + basis: businesses.length + ? `${businesses.length} CoinPay businesses, no per-business split in the snapshot` + : "CoinPay reported no businesses", + }; + } + const only = businesses[0] as { id?: string; name?: string }; + const name = String(only?.name ?? ""); + if (!(sameProperty(site, name) || site.site.toLowerCase() === name.toLowerCase())) { + return { usd: null, basis: `all commission belongs to ${name || "another business"}` }; + } + return { usd: num(roi.revenue.windowUsd), basis: `all CoinPay commission (${name})` }; +} + +/** Ad money and delivery for one property, joined exactly rather than shared out. */ +export function adMoneyForSite( + ads: AdsInput | null, + site: SiteLike, +): { earnedUsd: number; spentUsd: number; impressions: number; clicks: number } { + const model = (ads ?? {}) as AdsInput & { + slots?: Array<{ projectId?: string; name?: string; earnedCents?: number; impressions?: number; clicks?: number }>; + campaigns?: Array<{ url?: string | null; spentCents?: number }>; + }; + const out = { earnedUsd: 0, spentUsd: 0, impressions: 0, clicks: 0 }; + + for (const slot of model.slots ?? []) { + const mine = site.id ? slot.projectId === site.id : slot.name === site.site; + if (!mine) continue; + out.earnedUsd += cents(slot.earnedCents); + out.impressions += num(slot.impressions); + out.clicks += num(slot.clicks); + } + for (const campaign of model.campaigns ?? []) { + if (!sameProperty(site, campaign.url)) continue; + out.spentUsd += cents(campaign.spentCents); + } + return out; +} + +export type BuildSiteDetailInput = { + site: SiteLike; + roi: RoiModel; + ads: AdsInput | null; + finance: FinanceInput | null; + window: { range: string; who: string; financeDays: number }; +}; + +export function buildSiteDetail(input: BuildSiteDetailInput): SiteDetail { + const { site, roi } = input; + const gaps: string[] = []; + + const series = site.series ?? []; + const mix = site.mix; + // Under `who=all` a filtered read was never made, so the mix IS the series; + // under humans or bots it is the second, unfiltered read the API adds. When + // neither is there the share is unknown, and unknown is not 100%. + const mixKnown = Boolean(mix && mix.humans + mix.bots > 0); + const humans = mixKnown ? num(mix?.humans) : series.reduce((t, p) => t + num(p.humans), 0); + const bots = mixKnown ? num(mix?.bots) : 0; + const aiReferrals = mixKnown ? num(mix?.ai) : series.reduce((t, p) => t + num(p.ai), 0); + const humanShare = mixKnown && humans + bots > 0 ? humans / (humans + bots) : null; + + if (!series.length && !site.error) { + gaps.push("No series for this window, so momentum and volatility are unscored."); + } + if (!mixKnown && !site.error) { + gaps.push("The humans-against-bots mix is missing; press b for All to see both sides."); + } + + const visitShare = share(site.visitors, roi.attention.visitors); + const viewShare = share(site.pageviews, roi.attention.pageviews); + const costWindow = num(roi.cost.windowUsd); + const costByViews = roi.attention.pageviews > 0 ? costWindow * viewShare : null; + const costByVisits = roi.attention.visitors > 0 ? costWindow * visitShare : null; + + const ad = adMoneyForSite(input.ads, site); + const revenue = coinpayRevenueForSite(input.finance, roi, site); + if (revenue.usd === null) { + gaps.push(`Revenue is not attributable to one domain here: ${revenue.basis}.`); + } + + // The earn rail is a network-wide pool — crawler pass revenue funds it with + // the pass payment's own ref and no project column — so there is no per-domain + // figure to show. Named rather than omitted, because a missing money line on + // a money screen reads as a zero. + gaps.push("Earn-rail rewards are pooled network-wide; there is no per-domain share to report."); + + // Human visits are the denominator for a per-reader figure, and the score + // half that pays attention to money uses the same one. Ad earnings are + // internal and deliberately excluded: see the two rules in roi.ts. + const rpmUsd = humans > 0 && revenue.usd !== null ? (revenue.usd * 1000) / humans : null; + + // A site whose stats call failed is not scored at all. Its sources list may + // still hold something from a previous shape, and scoring off half an answer + // would rank a site we could not reach against sites we could. + // + // Otherwise: momentum and volatility read the shape of the series, which is + // filtered to whichever side was asked for; humanity, money and the sample + // floor read the unfiltered totals, the only place both sides are counted. + const score = site.error + ? scoreSite({ humans: [], sources: [], mixKnown: false }) + : scoreSite({ + humans: series.map((p) => num(p.humans)), + bots: series.map((p) => num(p.bots)), + sources: site.sources ?? [], + revenueUsd: revenue.usd ?? 0, + mixKnown, + ...(mixKnown ? { humansTotal: humans, botsTotal: bots } : {}), + }); + if (site.error) { + gaps.push("This site did not answer, so it is unscored rather than scored zero."); + } + + if (input.window.who === "bots") { + gaps.push("A bots-only window has no human series, so momentum and volatility are unscored."); + } + + return { + site: site.site, + url: site.url ?? null, + error: site.error ?? null, + window: { + range: input.window.range, + who: input.window.who, + days: roi.window.days, + financeDays: input.window.financeDays, + }, + traffic: { + visitors: num(site.visitors), + pageviews: num(site.pageviews), + humans, + bots, + aiReferrals, + humanShare, + mixKnown, + visitShare, + viewShare, + series, + sources: site.sources ?? [], + referrers: site.referrers ?? [], + pages: site.pages ?? [], + }, + money: { + costByViewsUsd: costByViews, + costByVisitsUsd: costByVisits, + adEarnedUsd: ad.earnedUsd, + adSpentUsd: ad.spentUsd, + adImpressions: ad.impressions, + adClicks: ad.clicks, + revenueUsd: revenue.usd, + revenueBasis: revenue.basis, + rpmUsd, + netUsd: revenue.usd === null || costByViews === null ? null : revenue.usd - costByViews, + }, + score, + gaps, + }; +} diff --git a/lib/tracker/apiStats.ts b/lib/tracker/apiStats.ts index f1f67d1..b475ed4 100644 --- a/lib/tracker/apiStats.ts +++ b/lib/tracker/apiStats.ts @@ -10,7 +10,14 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { hostOf } from "@/lib/ads/slots"; -import { fetchPanels, type ListItem, type PanelKey, type PanelPayload } from "@/lib/tracker/panels"; +import { + fetchPanel, + fetchPanels, + resolveDays, + type ListItem, + type PanelKey, + type PanelPayload, +} from "@/lib/tracker/panels"; import type { TrackerRange } from "@/lib/tracker/ranges"; import type { TrackerKind } from "@/lib/tracker/humans"; @@ -88,6 +95,15 @@ export async function resolveProject(sb: Sb, userId: string, site: string | null return { ok: false, status: 404, error: `No project for "${site}". Yours: ${projects.map((p) => p.name).join(", ")}` }; } +/** One bucket of the series, trimmed to what a client can plot or score. */ +export type StatsPoint = { + date: string; + pageviews: number; + humans: number; + bots: number; + ai: number; +}; + export type StatsAnswer = { project: { id: string; name: string; url: string }; range: string; @@ -96,10 +112,61 @@ export type StatsAnswer = { sources: ListItem[]; referrers: ListItem[]; pages: ListItem[]; + /** + * The shape over time, present only with `detail`. It is what the totals were + * summed from, so asking for it costs nothing extra. + */ + series?: StatsPoint[]; + /** + * Humans against bots over the same window, unfiltered. + * + * A filtered answer cannot carry this: asking for `who=humans` filters the + * RPC to `p_kind = 'human'`, so its bot column is zero by construction rather + * than by observation, and a share computed from it would read 100% human on + * a site that is 99% crawler. So this is a second, unfiltered read — skipped + * when the caller already asked for everything, where the main series IS it. + */ + mix?: { humans: number; bots: number; ai: number; events: number }; }; const asList = (payload: PanelPayload | undefined): ListItem[] => (Array.isArray(payload) ? payload : []); +const count = (v: unknown): number => { + const x = Number(v); + return Number.isFinite(x) ? x : 0; +}; + +/** The series payload as plottable points. Empty for a list payload or nothing. */ +export function seriesPoints(payload: PanelPayload | undefined): StatsPoint[] { + if (!payload || Array.isArray(payload)) return []; + const points = (payload as { points?: Record[] }).points ?? []; + return points.map((p) => ({ + date: String(p.date ?? ""), + pageviews: count(p.pageviews), + humans: count(p.humans), + bots: count(p.bots), + ai: count(p.ai), + })); +} + +/** Sum a series payload into the human / bot split. */ +export function mixFromSeries(payload: PanelPayload | undefined): { + humans: number; + bots: number; + ai: number; + events: number; +} { + const mix = { humans: 0, bots: 0, ai: 0, events: 0 }; + if (!payload || Array.isArray(payload)) return mix; + for (const p of (payload as { points?: Record[] }).points ?? []) { + mix.humans += count(p.humans); + mix.bots += count(p.bots); + mix.ai += count(p.ai); + mix.events += count(p.events); + } + return mix; +} + /** Sum a series payload's points into the two numbers a summary line needs. */ export function totalsFromSeries(payload: PanelPayload | undefined): { visitors: number; pageviews: number } { if (!payload || Array.isArray(payload)) return { visitors: 0, pageviews: 0 }; @@ -119,9 +186,19 @@ export async function projectStats( range: TrackerRange, kind: TrackerKind | null, who: string, + /** Add the series and the unfiltered human / bot mix. One extra RPC at most. */ + detail = false, ): Promise { - const panels = await fetchPanels(sb, project.id, STATS_PANELS, range, kind); - return { + const [panels, mixSeries] = await Promise.all([ + fetchPanels(sb, project.id, STATS_PANELS, range, kind), + // Only when the answer is filtered: at kind null the main series already is + // the unfiltered one, and a second identical query would be a second query. + detail && kind !== null + ? fetchPanel(sb, project.id, "series", range, await resolveDays(sb, project.id, range), null) + : Promise.resolve(undefined), + ]); + + const answer: StatsAnswer = { project: { id: project.id, name: project.name, url: project.url }, range: range.key, who, @@ -130,4 +207,11 @@ export async function projectStats( referrers: asList(panels.referrers), pages: asList(panels.pages), }; + if (!detail) return answer; + + return { + ...answer, + series: seriesPoints(panels.series), + mix: mixFromSeries(kind === null ? panels.series : mixSeries), + }; } diff --git a/packages/cli/README.md b/packages/cli/README.md index 4a5026d..6e4c16b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -14,7 +14,7 @@ and CoinPay for what the bank actually did. | Screen | Answers | | --- | --- | | ROI | Monthly burn against revenue, cost per reader, break-even | -| Traffic | Every site on the account, ranked, with its share of the cost | +| Traffic | Every site on the account, ranked, with its score and its share of the cost — open one for its own numbers | | Ads | Delivery as advertiser and as publisher, and ad-driven arrivals | | Money | Earnings, bank position, invoices, income vs spending by month | | Spend | Who we pay, largest first, and burn by category | @@ -22,6 +22,47 @@ and CoinPay for what the bank actually did. `1`–`5` or Tab switches screens, `w` cycles the window, `b` cycles humans / all / bots, `r` refreshes, `?` explains the arithmetic, `q` quits. +## One property at a time + +On **Traffic**, `↑`/`↓` pick a site and `Enter` — or a click on the row — opens +it. That screen is only that domain: its pageviews, visits, humans against +bots, AI referrals and where they arrived from; the burn prorated onto it by +both denominators; its ad earnings and spend, joined by project and by where +the campaign points; and the commission, when there is exactly one merchant +business to attribute it to. `Esc`, `←` or `2` comes back to the list. + +## The risk-to-viral score + +Every property gets a score out of 100, shown as a column on the list and taken +apart on the domain screen. It is arithmetic over numbers already on the screen, +not a model: + +``` +score = 100 × viral × (1 − risk/2) + +viral = momentum .40 + discovery .30 + humanity .20 + money .10 +risk = volatility .40 + concentration .30 + bot dependence .20 + unmonetised .10 +``` + +| Component | What it is | +| --- | --- | +| momentum | Human visits in the recent half of the window against the earlier half. Flat scores 0.5, doubling scores 1. | +| discovery | Share of arrivals through search, social, an AI assistant, an ad or another site's link, rather than direct. | +| humanity | Humans over humans plus bots, from an unfiltered read — never from a filtered one, whose bot column is zero by construction. | +| money | Revenue per 1,000 human visits against a $2 target. | +| volatility | Coefficient of variation of the human series. Scale-free, so a small site is not penalised for being small. | +| concentration | The largest single arrival channel's share. An even spread is not a risk; one channel being everything is. | +| bot dependence | 1 − humanity. | +| unmonetised | 1 − money. | + +A component with no data behind it is **dropped and its weight redistributed**, +never counted as a zero, and the domain screen prints the raw figure under each +one. A trailing `~` marks fewer than 25 human visits in the window — too small a +sample to lean on. A site whose stats call failed is not scored at all. + +`s` cycles the order: score, visitors, pageviews. `--sort=score` starts there. +`--json` carries `.sites[].score` with every component, for a script. + ## Two rules the numbers keep **Self-deal is not revenue.** Where an account advertises on its own slots, ad @@ -48,7 +89,7 @@ per-pageview figure instead. ``` crawlproof dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--interval=60] [--sites=a.com,b.com] [--concurrency=8] - [--no-coinpay] [--json] + [--sort=score|visitors|pageviews] [--no-coinpay] [--json] crawlproof stats [site] [--range=1d] [--who=humans] [--json] ``` diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 35e55dd..f434a9f 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -98,12 +98,34 @@ USAGE COMMANDS dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--interval=60] - [--sites=a.com,b.com] [--concurrency=8] [--no-coinpay] [--json] + [--sites=a.com,b.com] [--sort=score|visitors|pageviews] + [--concurrency=8] [--no-coinpay] [--json] A live dashboard of traffic across every site on the account, ad delivery, and — when a CoinPay merchant session is on the box — the bank feed behind it. Five screens: ROI, Traffic, Ads, Money, Spend. Aliases: roi, tui. --json prints the same snapshot for a script. + On Traffic, ↑/↓ pick a property and Enter — or a click — opens it: that + one domain's traffic and money, and every part of its score. Esc, ← or 2 + comes back. s cycles the order: score, visitors, pageviews. + + Risk-to-viral score + Each property is scored out of 100 from data already on the screen: + + score = 100 × viral × (1 − risk/2) + viral = momentum .40 + discovery .30 + humanity .20 + money .10 + risk = volatility .40 + concentration .30 + bots .20 + unmonetised .10 + + momentum is human visits in the recent half of the window against the + earlier half; discovery is the share arriving through search, social, AI, + an ad or a link rather than direct; humanity is humans over humans plus + bots; money is revenue per 1,000 human visits against $2. volatility is + the series' coefficient of variation, concentration is the largest single + channel's share. A component with no data is dropped and its weight + redistributed — never counted as a zero — and a trailing ~ marks a sample + under 25 human visits. The domain screen shows every component and the + number behind it. + stats [site] [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--json] Who arrived and from where: sources, referrers and top pages. Defaults to the last day and humans only, because a launch is invisible inside a @@ -215,6 +237,7 @@ async function cmdDashboard(args: Args): Promise { concurrency: Number(args.flags.concurrency) || 8, coinpay, only, + sort: args.flags.sort as string | undefined, theme: args.flags.theme as string | undefined, }); return 0; diff --git a/tests/dashboard-score.test.ts b/tests/dashboard-score.test.ts new file mode 100644 index 0000000..9c5e6bb --- /dev/null +++ b/tests/dashboard-score.test.ts @@ -0,0 +1,254 @@ +/** + * The risk-to-viral score. + * + * Two things are being pinned here. The first is that the arithmetic means what + * the comment in lib/dashboard/score.ts says it means — growth raises a score, + * crawler traffic lowers it, one channel carrying everything is a risk. The + * second is that none of it can produce a NaN, an Infinity or a confident zero + * from missing data, because a scoreboard that ranks a dead property above a + * live one is worse than no scoreboard. + */ +import { describe, expect, it } from "vitest"; + +import { + CONCENTRATION_FLOOR, + MIN_SAMPLE_HUMANS, + RISK_WEIGHTS, + TARGET_RPM_USD, + VIRAL_WEIGHTS, + clamp01, + coefficientOfVariation, + discoveryShare, + growthRate, + scoreSite, + topSourceShare, +} from "@/lib/dashboard/score"; + +const flat = (value: number, length = 14) => Array.from({ length }, () => value); + +const component = (list: { key: string; value: number | null }[], key: string) => + list.find((c) => c.key === key); + +describe("weights", () => { + it("each half sums to one, so the weighted mean is a mean", () => { + const sum = (w: Record) => Object.values(w).reduce((a, b) => a + b, 0); + expect(sum(VIRAL_WEIGHTS)).toBeCloseTo(1); + expect(sum(RISK_WEIGHTS)).toBeCloseTo(1); + }); +}); + +describe("clamp01", () => { + it("orders NaN to zero rather than propagating it", () => { + expect(clamp01(Number.NaN)).toBe(0); + expect(clamp01(Number.POSITIVE_INFINITY)).toBe(1); + expect(clamp01(-3)).toBe(0); + expect(clamp01(0.4)).toBe(0.4); + }); +}); + +describe("growthRate", () => { + it("compares the recent half of the window with the earlier half", () => { + expect(growthRate([1, 1, 2, 2])).toBeCloseTo(1); + expect(growthRate([2, 2, 1, 1])).toBeCloseTo(-0.5); + expect(growthRate([5, 5, 5, 5])).toBeCloseTo(0); + }); + + it("treats growth from nothing as growth rather than dividing by zero", () => { + const g = growthRate([0, 0, 3, 4]); + expect(g).not.toBeNull(); + expect(Number.isFinite(g as number)).toBe(true); + expect(g).toBeGreaterThan(0); + }); + + it("is null with too few buckets, or with no traffic at all", () => { + expect(growthRate([1, 2, 3])).toBeNull(); + expect(growthRate([])).toBeNull(); + expect(growthRate([0, 0, 0, 0])).toBeNull(); + }); +}); + +describe("coefficientOfVariation", () => { + it("is zero for a flat series and larger for a spiky one", () => { + expect(coefficientOfVariation(flat(10))).toBeCloseTo(0); + const spiky = coefficientOfVariation([0, 0, 0, 0, 0, 100]); + expect(spiky).not.toBeNull(); + expect(spiky as number).toBeGreaterThan(1); + }); + + it("is scale-free: ten visits a day and ten thousand score the same", () => { + const small = coefficientOfVariation([1, 3, 2, 4, 2, 3]); + const large = coefficientOfVariation([1000, 3000, 2000, 4000, 2000, 3000]); + expect(small).toBeCloseTo(large as number); + }); + + it("is null with no mean to divide by", () => { + expect(coefficientOfVariation([0, 0, 0])).toBeNull(); + expect(coefficientOfVariation([5, 5])).toBeNull(); + }); +}); + +describe("discoveryShare", () => { + it("counts channels a stranger can arrive through, not the ones we had", () => { + const share = discoveryShare([ + { label: "Search · google", value: 40 }, + { label: "Social · reddit", value: 10 }, + { label: "Direct", value: 50 }, + ]); + expect(share).toBeCloseTo(0.5); + }); + + it("reads the bucket spelling as well as the rendered label", () => { + expect(discoveryShare([{ label: "ai_referral:chatgpt", value: 3 }])).toBeCloseTo(1); + expect(discoveryShare([{ label: "AI · chatgpt", value: 3 }])).toBeCloseTo(1); + }); + + it("is null rather than zero when nothing arrived", () => { + expect(discoveryShare([])).toBeNull(); + expect(discoveryShare(undefined)).toBeNull(); + expect(discoveryShare([{ label: "Direct", value: 0 }])).toBeNull(); + }); +}); + +describe("topSourceShare", () => { + it("is the largest channel's share of everything", () => { + expect(topSourceShare([{ label: "a", value: 9 }, { label: "b", value: 1 }])).toBeCloseTo(0.9); + expect(topSourceShare([{ label: "a", value: 1 }])).toBeCloseTo(1); + expect(topSourceShare([])).toBeNull(); + }); +}); + +describe("scoreSite", () => { + const growing = () => ({ + humans: [5, 5, 6, 6, 10, 12, 14, 16], + bots: [1, 1, 1, 1, 1, 1, 1, 1], + sources: [ + { label: "Search · google", value: 40 }, + { label: "Social · reddit", value: 30 }, + { label: "Direct", value: 30 }, + ], + mixKnown: true, + }); + + it("scores a growing, human, well-spread property above a shrinking one", () => { + const up = scoreSite(growing()); + const down = scoreSite({ ...growing(), humans: [16, 14, 12, 10, 6, 6, 5, 5] }); + expect(up.score).not.toBeNull(); + expect(down.score).not.toBeNull(); + expect(up.score as number).toBeGreaterThan(down.score as number); + }); + + it("keeps every score inside 0..100", () => { + for (const input of [ + growing(), + { humans: [] }, + { humans: flat(0) }, + { humans: [1e12, 1e12, 0, 0], bots: flat(1e12), mixKnown: true }, + { humans: flat(5), revenueUsd: 1e9, mixKnown: true }, + ]) { + const model = scoreSite(input); + if (model.score === null) continue; + expect(model.score).toBeGreaterThanOrEqual(0); + expect(model.score).toBeLessThanOrEqual(100); + } + }); + + it("marks a mostly-crawler property down through humanity and bot dependence", () => { + const human = scoreSite(growing()); + const crawled = scoreSite({ ...growing(), bots: [500, 500, 500, 500, 500, 500, 500, 500] }); + expect(component(crawled.viralComponents, "humanity")?.value as number).toBeLessThan(0.1); + expect(component(crawled.riskComponents, "botDependence")?.value as number).toBeGreaterThan(0.9); + expect(crawled.score as number).toBeLessThan(human.score as number); + }); + + it("does not assume a 100% human property when the mix was never measured", () => { + const model = scoreSite({ ...growing(), bots: [], mixKnown: false }); + expect(component(model.viralComponents, "humanity")?.value).toBeNull(); + expect(component(model.riskComponents, "botDependence")?.value).toBeNull(); + expect(model.notes.join(" ")).toMatch(/unknown/i); + // Dropped, not zeroed: the remaining weight still adds up to a mean. + expect(model.coverage).toBeLessThan(1); + expect(model.score).not.toBeNull(); + }); + + it("treats one channel carrying everything as a risk, an even split as none", () => { + const oneChannel = scoreSite({ + ...growing(), + sources: [{ label: "Search · google", value: 100 }], + }); + const spread = scoreSite({ + ...growing(), + sources: [ + { label: "Search · google", value: 34 }, + { label: "Social · reddit", value: 33 }, + { label: "Referral · news.ycombinator.com", value: 33 }, + ], + }); + expect(component(oneChannel.riskComponents, "concentration")?.value).toBeCloseTo(1); + expect(component(spread.riskComponents, "concentration")?.value).toBeCloseTo(0); + expect(spread.score as number).toBeGreaterThan(oneChannel.score as number); + }); + + it("puts the concentration floor where an even-ish split stops counting", () => { + const atFloor = scoreSite({ + ...growing(), + sources: [ + { label: "Search · google", value: CONCENTRATION_FLOOR * 100 }, + { label: "Direct", value: (1 - CONCENTRATION_FLOOR) * 100 }, + ], + }); + expect(component(atFloor.riskComponents, "concentration")?.value).toBeCloseTo(0); + }); + + it("scores money against the target RPM and caps it there", () => { + const humans = 1000; + const atTarget = scoreSite({ humans: flat(humans / 14), revenueUsd: TARGET_RPM_USD, mixKnown: true }); + const over = scoreSite({ humans: flat(humans / 14), revenueUsd: TARGET_RPM_USD * 50, mixKnown: true }); + expect(component(atTarget.viralComponents, "money")?.value).toBeCloseTo(1, 1); + expect(component(over.viralComponents, "money")?.value).toBe(1); + expect(component(over.riskComponents, "unmonetised")?.value).toBe(0); + }); + + it("flags a sample too small to lean on without hiding the number", () => { + const tiny = scoreSite({ humans: [1, 1, 2, 1], mixKnown: true, bots: [0, 0, 0, 0] }); + expect(tiny.provisional).toBe(true); + expect(tiny.humans).toBeLessThan(MIN_SAMPLE_HUMANS); + expect(tiny.score).not.toBeNull(); + expect(tiny.notes.join(" ")).toContain(String(MIN_SAMPLE_HUMANS)); + }); + + it("prefers the unfiltered totals when the series only counted one side", () => { + // What `who=bots` looks like: the human column is zero by construction, and + // the mix knows the real split. + const model = scoreSite({ + humans: flat(0), + bots: flat(10), + humansTotal: 900, + botsTotal: 100, + mixKnown: true, + }); + expect(model.humans).toBe(900); + expect(component(model.viralComponents, "humanity")?.value).toBeCloseTo(0.9); + expect(model.provisional).toBe(false); + }); + + it("returns no score at all, rather than a zero, for a property with no data", () => { + const empty = scoreSite({ humans: [] }); + expect(empty.score).toBeNull(); + expect(empty.viral).toBe(0); + expect(empty.risk).toBe(0); + expect(empty.coverage).toBe(0); + }); + + it("survives junk without a NaN reaching a component", () => { + const model = scoreSite({ + humans: [Number.NaN, Number.POSITIVE_INFINITY, 3, 4] as number[], + bots: ["7" as unknown as number, null as unknown as number, 1, 1], + sources: [{ label: "Search · google", value: Number.NaN }, { label: "", value: 5 }] as never, + revenueUsd: Number.NaN, + mixKnown: true, + }); + const values = [...model.viralComponents, ...model.riskComponents].map((c) => c.value); + for (const v of values) expect(v === null || Number.isFinite(v)).toBe(true); + expect(model.score === null || Number.isFinite(model.score)).toBe(true); + }); +}); diff --git a/tests/dashboard-screens.test.ts b/tests/dashboard-screens.test.ts new file mode 100644 index 0000000..8f2f7f8 --- /dev/null +++ b/tests/dashboard-screens.test.ts @@ -0,0 +1,248 @@ +/** + * What the dashboard actually draws, and how you get from the list to a domain. + * + * Rendered rather than inspected: hqtui screens are only observable by drawing + * them, and the two things that bite — a panel header that drops half of itself + * at one width, and a table that is unreachable by the mouse because nothing + * registered a hit region — are invisible in source review. + * + * The navigation is tested through `handleKey`, the same function the app binds + * its key handler to, so what passes here is what the terminal does. + */ +import { describe, expect, it } from "vitest"; +import { renderToScreen } from "@profullstack/hqtui/testing"; + +import { buildRoi } from "@/lib/dashboard/roi"; +import { buildSiteDetail } from "@/lib/dashboard/site"; +import type { DashboardSnapshot, SiteStats } from "@/lib/dashboard/collect"; +import { handleKey, initialState, renderBody, scoreText, sortSites, type State } from "@/cli/dashboard"; + +const WIDE = { width: 160, height: 44 }; + +function siteRow(over: Partial = {}): SiteStats { + return { + site: "nichedb.dev", + id: "p-1", + url: "https://nichedb.dev/", + visitors: 400, + pageviews: 120, + sources: [ + { label: "Search · google", value: 70 }, + { label: "Direct", value: 30 }, + ], + referrers: [{ label: "news.ycombinator.com", value: 9 }], + pages: [{ label: "/pricing", value: 40 }], + series: Array.from({ length: 8 }, (_, i) => ({ + date: `2026-09-0${i + 1}`, + pageviews: 10 + i * 3, + humans: 10 + i * 3, + bots: 2, + ai: 1, + })), + mix: { humans: 164, bots: 20, ai: 8, events: 184 }, + ...over, + }; +} + +function snapshot(sites: SiteStats[]): DashboardSnapshot { + const finance = { + windowDays: 30, + businesses: [{ id: "b-1", name: "coinpayportal.com" }], + earnings: { commissionUsd: 500, grossVolumeUsd: 50_000 }, + series: Array.from({ length: 30 }, () => ({ volumeUsd: 100, commissionUsd: 1 })), + position: { + lookbackDays: 180, + monthsObserved: 6, + spending: { perMonth: 4_000 }, + scopes: [{ scope: "business", spending: 24_000, income: 600, accounts: 1 }], + }, + bank: { accounts: [], ledger: [], ledgerTotal: 0 }, + }; + const ads = { + rangeDays: 30, + totals: { spentCents: 500, earnedCents: 700, pubImpressions: 9_000, pubClicks: 12 }, + slots: [{ id: "s-1", name: "nichedb.dev", projectId: "p-1", impressions: 900, clicks: 3, earnedCents: 250 }], + campaigns: [{ id: "c-1", name: "NicheDB", url: "https://nichedb.dev/", spentCents: 120 }], + }; + const roi = buildRoi({ + traffic: { range: "1m", who: "humans", sites }, + ads, + finance, + }); + const window = { range: "1m", who: "humans", financeDays: 30 }; + for (const site of sites) { + site.score = buildSiteDetail({ site, roi, ads, finance, window }).score; + } + return { + generatedAt: new Date().toISOString(), + window, + sites, + fleet: { sources: sites[0]?.sources ?? [], referrers: [], pages: sites[0]?.pages ?? [] }, + ads, + finance, + roi, + errors: {}, + }; +} + +function stateWith(over: Partial = {}, sites: SiteStats[] = [siteRow()]): State { + return initialState({ tab: 1, range: "1m", snapshot: snapshot(sites), ...over }); +} + +const draw = (state: State) => + renderToScreen(({ ui, theme }) => renderBody(ui, state, theme), WIDE); + +describe("Traffic list", () => { + it("draws a Score column beside the traffic, with the site's score in it", () => { + const state = stateWith(); + const text = draw(state).text(); + expect(text).toContain("Score"); + expect(text).toContain("nichedb.dev"); + expect(text).toContain(scoreText(state.snapshot?.sites[0] as SiteStats)); + }); + + it("says which order it is in and how to change it", () => { + const text = draw(stateWith()).text(); + expect(text).toMatch(/by score/); + expect(text).toMatch(/Enter opens/); + }); + + it("registers a mouse region, so a click can reach a row at all", () => { + const screen = draw(stateWith()); + expect(screen.regions.length).toBeGreaterThan(0); + expect(screen.regions.some((r) => typeof r.onClick === "function")).toBe(true); + }); + + it("marks a site that did not answer rather than drawing it as a quiet one", () => { + const rows = [siteRow(), siteRow({ site: "down.dev", id: "p-2", url: "https://down.dev", error: "504 Gateway Timeout", visitors: 0, pageviews: 0, series: [], mix: undefined })]; + const text = draw(stateWith({}, rows)).text(); + expect(text).toContain("504"); + expect(text).toContain("down.dev"); + }); +}); + +describe("sortSites", () => { + const rows = () => [ + siteRow({ site: "busy.dev", id: "p-2", url: "https://busy.dev", visitors: 90_000, pageviews: 12 }), + siteRow(), + siteRow({ site: "dead.dev", id: "p-3", url: "https://dead.dev", error: "timeout", visitors: 0, pageviews: 0, series: [], mix: undefined }), + ]; + + it("puts the best score first, and a site that did not answer last", () => { + const sorted = sortSites(snapshot(rows()).sites, "score"); + expect(sorted[sorted.length - 1]?.site).toBe("dead.dev"); + expect(sorted[0]?.score?.score).not.toBeNull(); + }); + + it("ranks the busiest site first by visitors and not necessarily by score", () => { + const sites = snapshot(rows()).sites; + expect(sortSites(sites, "visitors")[0]?.site).toBe("busy.dev"); + expect(sortSites(sites, "pageviews")[0]?.site).toBe("nichedb.dev"); + }); +}); + +describe("opening a domain", () => { + const noop = { refresh: () => {} }; + + it("Enter on the list opens the selected property", () => { + const state = stateWith(); + expect(handleKey(state, { name: "enter" }, noop)).toBe(true); + expect(state.domain).toBe("nichedb.dev"); + }); + + it("Esc, ← and 2 all come back to the list", () => { + for (const key of ["escape", "backspace", "left", "2"]) { + const state = stateWith({ domain: "nichedb.dev" }); + expect(handleKey(state, { name: key }, noop)).toBe(true); + expect(state.domain, key).toBeNull(); + } + }); + + it("Esc on the list itself is not swallowed", () => { + expect(handleKey(stateWith(), { name: "escape" }, noop)).toBe(false); + }); + + it("↑/↓ move the selection rather than only the scroll", () => { + const rows = [siteRow(), siteRow({ site: "second.dev", id: "p-2", url: "https://second.dev" })]; + const state = stateWith({ sort: "visitors" }, rows); + draw(state); // the pane learns how many rows there are by being drawn + handleKey(state, { name: "down" }, noop); + expect(state.panes.sites?.selected).toBe(1); + handleKey(state, { name: "enter" }, noop); + expect(state.domain).toBe(sortSites(state.snapshot?.sites ?? [], "visitors")[1]?.site); + }); + + it("keeps the highlight on the same property when the order changes", () => { + const rows = [ + siteRow({ site: "busy.dev", id: "p-2", url: "https://busy.dev", visitors: 90_000, pageviews: 5 }), + siteRow(), + ]; + const state = stateWith({ sort: "visitors" }, rows); + draw(state); + handleKey(state, { name: "down" }, noop); + const held = sortSites(state.snapshot?.sites ?? [], "visitors")[1]?.site; + handleKey(state, { name: "s" }, noop); + const nowAt = sortSites(state.snapshot?.sites ?? [], state.sort)[state.panes.sites?.selected ?? 0]?.site; + expect(nowAt).toBe(held); + }); +}); + +describe("the domain screen", () => { + const text = (over: Partial = {}) => + draw(stateWith({ domain: "nichedb.dev" }, [siteRow(over)])).text(); + + it("leads with the domain and its own traffic", () => { + const out = text(); + expect(out).toContain("nichedb.dev"); + expect(out).toContain("Pageviews"); + expect(out).toContain("Human share"); + expect(out).toContain("AI referrals"); + }); + + it("shows the money for that domain, cost by both denominators", () => { + const out = text(); + expect(out).toContain("Cost · by views"); + expect(out).toContain("Cost · by visits"); + expect(out).toContain("Ad earned"); + }); + + it("shows the score and every component behind it", () => { + const out = text(); + expect(out).toContain("Risk-to-viral"); + expect(out).toContain("viral"); + expect(out).toContain("risk"); + for (const part of ["Momentum", "Discovery", "Humanity", "Money", "Volatility", "Bot dependence"]) { + expect(out, part).toContain(part); + } + // The formula itself, so nobody has to trust the number. + expect(out).toMatch(/100 × viral/); + }); + + it("prints a dash and a reason rather than a zero it cannot stand behind", () => { + const out = text(); + expect(out).toMatch(/Revenue\s+—/); + expect(out).toMatch(/Earn-rail/); + }); + + it("says so when the site is one that did not answer", () => { + const out = text({ error: "504 Gateway Timeout", series: [], mix: undefined, visitors: 0, pageviews: 0 }); + expect(out).toContain("504 Gateway Timeout"); + expect(out).toContain("missing, not zero"); + }); + + it("says so when the domain is no longer in the snapshot", () => { + const state = stateWith({ domain: "vanished.dev" }); + expect(draw(state).text()).toContain("not in the current snapshot"); + }); + + // hqtui draws a panel's title and its subtitle in the same border row and the + // subtitle wins, so a subtitle sized against the whole pane costs the panel + // its own name at exactly the widths nobody renders in a test fixture. + it("keeps every panel's name at a narrow terminal", () => { + const state = stateWith({ domain: "nichedb.dev" }); + const narrow = renderToScreen(({ ui, theme }) => renderBody(ui, state, theme), { width: 96, height: 30 }); + for (const title of ["nichedb.dev", "Money", "Risk-to-viral", "Why it scores that"]) { + expect(narrow.text(), title).toContain(title); + } + }); +}); diff --git a/tests/dashboard-site.test.ts b/tests/dashboard-site.test.ts new file mode 100644 index 0000000..6a74c89 --- /dev/null +++ b/tests/dashboard-site.test.ts @@ -0,0 +1,228 @@ +/** + * One property's own numbers. + * + * The joins are the whole point of this module and each of them is a different + * one — earnings by project id, spend by destination host, commission only when + * there is exactly one business to attribute it to — so each is pinned here, + * along with the refusals: what this cannot attribute it must say rather than + * divide out. + */ +import { describe, expect, it } from "vitest"; + +import { buildRoi, type AdsInput, type FinanceInput } from "@/lib/dashboard/roi"; +import { + adMoneyForSite, + buildSiteDetail, + coinpayRevenueForSite, + hostFrom, + sameProperty, + type SiteLike, +} from "@/lib/dashboard/site"; + +const site = (over: Partial = {}): SiteLike => ({ + site: "nichedb.dev", + id: "p-1", + url: "https://nichedb.dev/", + visitors: 400, + pageviews: 100, + sources: [ + { label: "Search · google", value: 60 }, + { label: "Direct", value: 40 }, + ], + referrers: [], + pages: [{ label: "/", value: 80 }], + series: Array.from({ length: 8 }, (_, i) => ({ + date: `2026-09-0${i + 1}`, + pageviews: 10 + i, + humans: 10 + i, + bots: 2, + ai: 1, + })), + mix: { humans: 116, bots: 16, ai: 8, events: 132 }, + ...over, +}); + +const ads = (): AdsInput => ({ + rangeDays: 30, + totals: { spentCents: 500, earnedCents: 700, pubImpressions: 10_000, pubClicks: 20 }, + slots: [ + { id: "s-1", name: "nichedb.dev", projectId: "p-1", impressions: 900, clicks: 3, earnedCents: 250 }, + { id: "s-2", name: "other.dev", projectId: "p-2", impressions: 50, clicks: 0, earnedCents: 400 }, + ], + campaigns: [ + { id: "c-1", name: "NicheDB", url: "https://www.nichedb.dev/pricing", spentCents: 120 }, + { id: "c-2", name: "Other", url: "https://other.dev/", spentCents: 999 }, + { id: "c-3", name: "Broken", url: null, spentCents: 777 }, + ], +}); + +const finance = (businesses: Array<{ id?: string; name?: string }>): FinanceInput => ({ + windowDays: 30, + businesses, + earnings: { commissionUsd: 900, grossVolumeUsd: 90_000 }, + series: Array.from({ length: 30 }, () => ({ volumeUsd: 100, commissionUsd: 1 })), + position: { + lookbackDays: 180, + monthsObserved: 6, + spending: { perMonth: 3_000 }, + scopes: [{ scope: "business", spending: 18_000, income: 600, accounts: 1 }], + }, + bank: { accounts: [], ledger: [], ledgerTotal: 0 }, +}); + +const roiFor = (sites: SiteLike[], fin: FinanceInput | null = finance([])) => + buildRoi({ + traffic: { + range: "1m", + who: "humans", + sites: sites.map((s) => ({ site: s.site, visitors: s.visitors, pageviews: s.pageviews })), + }, + ads: ads(), + finance: fin, + }); + +describe("hostFrom", () => { + it("takes a host out of a URL, a bare host, or neither", () => { + expect(hostFrom("https://www.NicheDB.dev/pricing?x=1")).toBe("nichedb.dev"); + expect(hostFrom("nichedb.dev")).toBe("nichedb.dev"); + expect(hostFrom("")).toBeNull(); + expect(hostFrom(null)).toBeNull(); + expect(hostFrom(" ")).toBeNull(); + }); +}); + +describe("sameProperty", () => { + it("matches on the project URL, on the project name, and ignores www", () => { + expect(sameProperty(site(), "https://www.nichedb.dev/a")).toBe(true); + expect(sameProperty(site({ url: undefined }), "https://nichedb.dev/")).toBe(true); + expect(sameProperty(site(), "https://other.dev/")).toBe(false); + expect(sameProperty(site(), null)).toBe(false); + }); +}); + +describe("adMoneyForSite", () => { + it("takes earnings by project id and spend by where the campaign points", () => { + const money = adMoneyForSite(ads(), site()); + expect(money.earnedUsd).toBeCloseTo(2.5); + expect(money.spentUsd).toBeCloseTo(1.2); + expect(money.impressions).toBe(900); + expect(money.clicks).toBe(3); + }); + + it("is all zeros, not a share of the fleet, when nothing matches", () => { + const money = adMoneyForSite(ads(), site({ id: "p-9", site: "nobody.dev", url: "https://nobody.dev" })); + expect(money).toEqual({ earnedUsd: 0, spentUsd: 0, impressions: 0, clicks: 0 }); + }); + + it("survives an absent ads feed", () => { + expect(adMoneyForSite(null, site()).earnedUsd).toBe(0); + }); +}); + +describe("coinpayRevenueForSite", () => { + it("attributes the whole commission when there is exactly one matching business", () => { + const fin = finance([{ id: "b-1", name: "nichedb.dev" }]); + const result = coinpayRevenueForSite(fin, roiFor([site()], fin), site()); + expect(result.usd).toBeGreaterThan(0); + expect(result.basis).toContain("nichedb.dev"); + }); + + it("refuses rather than sharing it out across several businesses", () => { + const fin = finance([{ name: "a" }, { name: "b" }]); + const result = coinpayRevenueForSite(fin, roiFor([site()], fin), site()); + expect(result.usd).toBeNull(); + expect(result.basis).toMatch(/no per-business split/); + }); + + it("refuses when the one business is a different property", () => { + const fin = finance([{ name: "coinpayportal.com" }]); + expect(coinpayRevenueForSite(fin, roiFor([site()], fin), site()).usd).toBeNull(); + }); + + it("says there is no session rather than reporting zero", () => { + expect(coinpayRevenueForSite(null, roiFor([site()], null), site()).basis).toMatch(/no CoinPay session/); + }); +}); + +describe("buildSiteDetail", () => { + const build = (over: Partial = {}, fin: FinanceInput | null = finance([])) => { + const rows = [site(over), site({ site: "other.dev", id: "p-2", url: "https://other.dev", visitors: 600, pageviews: 900 })]; + return buildSiteDetail({ + site: rows[0] as SiteLike, + roi: roiFor(rows, fin), + ads: ads(), + finance: fin, + window: { range: "1m", who: "humans", financeDays: 30 }, + }); + }; + + it("prorates cost by both denominators, because they disagree by orders of magnitude", () => { + const detail = build(); + // 100 of 1,000 pageviews, but 400 of 1,000 visits. + expect(detail.traffic.viewShare).toBeCloseTo(0.1); + expect(detail.traffic.visitShare).toBeCloseTo(0.4); + expect(detail.money.costByVisitsUsd as number).toBeGreaterThan(detail.money.costByViewsUsd as number); + }); + + it("reads the human / bot split from the unfiltered mix, not from the filtered series", () => { + const detail = build(); + expect(detail.traffic.mixKnown).toBe(true); + expect(detail.traffic.humans).toBe(116); + expect(detail.traffic.bots).toBe(16); + expect(detail.traffic.humanShare as number).toBeCloseTo(116 / 132); + }); + + it("says the mix is unknown rather than calling a site 100% human", () => { + const detail = build({ mix: undefined }); + expect(detail.traffic.mixKnown).toBe(false); + expect(detail.traffic.humanShare).toBeNull(); + expect(detail.gaps.join(" ")).toMatch(/mix is missing/i); + }); + + it("names the earn rail as pooled rather than leaving a money line blank", () => { + expect(build().gaps.join(" ")).toMatch(/Earn-rail/); + }); + + it("has no revenue, no RPM and no net when revenue cannot be attributed", () => { + const detail = build(); + expect(detail.money.revenueUsd).toBeNull(); + expect(detail.money.rpmUsd).toBeNull(); + expect(detail.money.netUsd).toBeNull(); + expect(detail.gaps.join(" ")).toMatch(/not attributable/); + }); + + it("computes revenue per 1k humans once there is revenue to divide", () => { + const fin = finance([{ name: "nichedb.dev" }]); + const detail = build({}, fin); + expect(detail.money.revenueUsd as number).toBeGreaterThan(0); + expect(detail.money.rpmUsd as number).toBeCloseTo(((detail.money.revenueUsd as number) * 1000) / 116); + expect(Number.isFinite(detail.money.netUsd as number)).toBe(true); + }); + + it("carries a failed site through as missing rather than as a quiet day", () => { + const detail = build({ error: "504 Gateway Timeout", series: [], mix: undefined, visitors: 0, pageviews: 0 }); + expect(detail.error).toBe("504 Gateway Timeout"); + expect(detail.score.score).toBeNull(); + }); + + it("scores the property and keeps it in range", () => { + const detail = build(); + expect(detail.score.score).not.toBeNull(); + expect(detail.score.score as number).toBeGreaterThanOrEqual(0); + expect(detail.score.score as number).toBeLessThanOrEqual(100); + expect(detail.score.viralComponents).toHaveLength(4); + expect(detail.score.riskComponents).toHaveLength(4); + }); + + it("says a bots-only window cannot carry momentum", () => { + const rows = [site()]; + const detail = buildSiteDetail({ + site: rows[0] as SiteLike, + roi: roiFor(rows), + ads: ads(), + finance: finance([]), + window: { range: "1m", who: "bots", financeDays: 30 }, + }); + expect(detail.gaps.join(" ")).toMatch(/bots-only/); + }); +}); diff --git a/tests/tracker-api-stats.test.ts b/tests/tracker-api-stats.test.ts index 7099958..b8985dc 100644 --- a/tests/tracker-api-stats.test.ts +++ b/tests/tracker-api-stats.test.ts @@ -8,7 +8,8 @@ */ import { describe, expect, it } from "vitest"; -import { resolveProject, totalsFromSeries } from "@/lib/tracker/apiStats"; +import { mixFromSeries, resolveProject, seriesPoints, totalsFromSeries } from "@/lib/tracker/apiStats"; +import { parseDetail } from "@/app/api/tracker/v1/stats/route"; type Row = { id: string; name: string; url: string }; @@ -97,3 +98,45 @@ describe("totalsFromSeries", () => { expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ visitors: 0, pageviews: 4 }); }); }); + +describe("seriesPoints", () => { + it("trims a series payload to what a client can plot", () => { + expect( + seriesPoints({ + points: [{ date: "2026-09-01", pageviews: "4", humans: 3, bots: 1, ai: 2, interactions: 9 }], + } as never), + ).toEqual([{ date: "2026-09-01", pageviews: 4, humans: 3, bots: 1, ai: 2 }]); + }); + + it("is empty for a list payload or nothing, rather than throwing", () => { + expect(seriesPoints(undefined)).toEqual([]); + expect(seriesPoints([] as never)).toEqual([]); + }); +}); + +describe("mixFromSeries", () => { + it("sums both sides, which is the only honest place a human share comes from", () => { + expect( + mixFromSeries({ + points: [ + { humans: 3, bots: 7, ai: 1, events: 10 }, + { humans: 2, bots: 8, ai: 0, events: 10 }, + ], + } as never), + ).toEqual({ humans: 5, bots: 15, ai: 1, events: 20 }); + }); + + it("is zeros for nothing at all, never NaN", () => { + expect(mixFromSeries(undefined)).toEqual({ humans: 0, bots: 0, ai: 0, events: 0 }); + }); +}); + +describe("parseDetail", () => { + it("only an affirmative buys the extra query", () => { + expect(parseDetail("1")).toBe(true); + expect(parseDetail("true")).toBe(true); + expect(parseDetail(null)).toBe(false); + expect(parseDetail("0")).toBe(false); + expect(parseDetail("; drop table")).toBe(false); + }); +});