From 8006bce1118dba77e345260aa1594a8a90138e32 Mon Sep 17 00:00:00 2001 From: damengrandom Date: Sat, 1 Aug 2026 14:26:56 +1000 Subject: [PATCH 1/2] fix: create the schema before every job, not only on setup CI never runs `pnpm run setup`, so a job shipping a new table hit "relation ... does not exist" in production. initDb is idempotent, so paying it per run is cheaper than the failure mode it prevents. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01F5CmYPdQcMeLzzJc3iDQ2u --- src/index.ts | 1 + src/storage/own-db.ts | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1021872..4c36b7f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,7 @@ async function main(): Promise { const job = findJobByCliArg(process.argv) if (job) { + await initDb() await job.run() process.exit(0) diff --git a/src/storage/own-db.ts b/src/storage/own-db.ts index d42e6c7..34cbdf9 100644 --- a/src/storage/own-db.ts +++ b/src/storage/own-db.ts @@ -64,11 +64,6 @@ export async function initDb(): Promise { ); `) - // The T15 AI news feature (removed in T20) left an ai_news table behind on any - // database it ever ran against — including production. CREATE TABLE IF NOT - // EXISTS silently keeps that older shape, so inserts fail on the missing - // columns until it is brought forward. Its `summary` held what is now - // `snippet`, so the rename preserves the old rows as dedupe history. await pool.query(` DO $$ BEGIN @@ -119,9 +114,6 @@ export async function saveCleanupLog(result: CleanupResult, tableName: string): ) } -// One row per repo. A repo that trends again updates in place — star counts stay -// current instead of going stale behind a dedup filter. created_at keeps its -// original value, so it still reads as "first seen". export async function saveTrendingRepos(repos: TrendingRepoLog[]): Promise { for (const repo of repos) { await pool.query( @@ -154,9 +146,6 @@ export async function saveTrendingRepos(repos: TrendingRepoLog[]): Promise } } -// One row per story, keyed on url. A story that resurfaces updates in place -// rather than inserting a duplicate — created_at keeps its original value, so it -// still reads as "first seen". export async function saveAiNews(items: AiNewsLog[]): Promise { for (const item of items) { await pool.query( From 0f1dfc8040968128b6dd6300ad8a2498e1e86c13 Mon Sep 17 00:00:00 2001 From: damengrandom Date: Sat, 1 Aug 2026 14:27:12 +1000 Subject: [PATCH 2/2] fix(T-29): bound the trending digest, and stop declaring unreachable LLM fallbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #18. One repo's tags could take down the whole digest. summary was capped at 140 chars in code, but tags — also model-written — had no bound on count or length, in the schema or the builder. Eight repos with one verbose tags array rendered at 4107 chars against Telegram's 4096 limit, and Telegram rejects an over-long message rather than truncating it, so all eight were lost and saved with sent=false. Same payload now renders at 2647 and delivers all eight. Fixed in three parts: - TRENDING_TAGS_MAX / TRENDING_TAG_MAX bound the tag list, and the existing magic 140 becomes TRENDING_SUMMARY_MAX. Only the model-written fields are capped; GitHub already bounds the scraped ones. - buildTrendingMessage drops repos from the end until the message fits. The per-field caps alone cannot guarantee this, because escaping runs after them and expands text — 140 '&' characters become 700. A short digest beats no digest. - The builder is extracted from the tool's func, mirroring buildAiNewsMessage, so it is testable without stubbing fetch. This file had no test coverage at all; it now has eight cases, including the reported payload and the escape-expansion path. Deliberately not added: a .max() on TrendingRepoOutputSchema.tags. curator.graph.ts validates the model's output against it inside a retry loop, so a cap there turns "returned six tags" into a validation failure and a wasted retry. Truncating in code degrades; rejecting in the schema does not. LLM_FALLBACK_MODELS declared three models but OpenRouter caps its `models` list at three entries including the primary, so createLlm's slice(0, 3) silently dropped the last one — configuration that read as a safety net and was unreachable. Cut to a single fallback, which is what the cap leaves room for. createLlm had no test; one now asserts every configured fallback actually reaches OpenRouter. Verified with a live run: 26 repos scraped, top 8 curated, Telegram digest delivered, 8 rows saved with sent=true. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01F5CmYPdQcMeLzzJc3iDQ2u --- src/agent/llm.test.ts | 29 ++++++++-- src/constants/index.ts | 30 +++------- src/tools/news-telegram.test.ts | 97 +++++++++++++++++++++++++++++++++ src/tools/news-telegram.tool.ts | 80 ++++++++++++++++----------- 4 files changed, 176 insertions(+), 60 deletions(-) create mode 100644 src/tools/news-telegram.test.ts diff --git a/src/agent/llm.test.ts b/src/agent/llm.test.ts index d6e08e8..d9048d5 100644 --- a/src/agent/llm.test.ts +++ b/src/agent/llm.test.ts @@ -1,6 +1,7 @@ import { test, afterEach } from 'node:test' import assert from 'node:assert/strict' -import { failLoudlyOnProviderError } from './llm.ts' +import { createLlm, failLoudlyOnProviderError } from './llm.ts' +import { DEFAULT_LLM, LLM_FALLBACK_MODELS } from '../constants/index.ts' const realFetch = globalThis.fetch @@ -14,9 +15,6 @@ function stubFetch(body: string, init: { status?: number; contentType?: string } globalThis.fetch = (async () => new Response(body, { status, headers: { 'content-type': contentType } })) as typeof fetch } -// The real payload that took the trending job down: HTTP 200, an `error` object, -// and no `choices`. The OpenAI SDK treats 200 as success and hands back an empty -// result, which only explodes later as `undefined.message`. test('throws on an HTTP 200 error body with no choices', async () => { stubFetch(JSON.stringify({ error: { message: 'Upstream error from Nvidia: ResourceExhausted (32/32)', code: 502 } })) @@ -44,8 +42,6 @@ test('passes a normal completion straight through', async () => { assert.deepEqual(await res.json(), { choices: [{ message: { content: 'OK' } }] }) }) -// A body carrying BOTH an error and choices is a partial success — the SDK can -// still read a completion out of it, so it must not be turned into a throw. test('does not throw when choices are present alongside an error', async () => { stubFetch(JSON.stringify({ error: { message: 'partial' }, choices: [{ message: { content: 'OK' } }] })) @@ -88,3 +84,24 @@ test('leaves the response body readable for the caller', async () => { assert.equal(parsed.choices[0].message.content, 'still here') }) + +test('every configured fallback actually reaches OpenRouter', () => { + const key = process.env.LLM_API_KEY + const model = process.env.LLM_MODEL + + process.env.LLM_API_KEY = 'dummy' + delete process.env.LLM_MODEL + + try { + const { modelKwargs } = createLlm() as unknown as { modelKwargs: { models: string[] } } + + assert.equal(modelKwargs.models[0], DEFAULT_LLM, 'primary must be tried first') + for (const fb of LLM_FALLBACK_MODELS) { + assert.ok(modelKwargs.models.includes(fb), `configured fallback silently dropped: ${fb}`) + } + } finally { + if (key !== undefined) process.env.LLM_API_KEY = key + else delete process.env.LLM_API_KEY + if (model !== undefined) process.env.LLM_MODEL = model + } +}) diff --git a/src/constants/index.ts b/src/constants/index.ts index 191435e..1869b7a 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -1,38 +1,22 @@ export const DEFAULT_LLM = 'nvidia/nemotron-3-ultra-550b-a55b:free' export const DEFAULT_LLM_BASE_URL = 'https://openrouter.ai/api/v1' -// Free, tool-capable models OpenRouter falls back to when the primary is at -// capacity. Order matters — first available wins. -export const LLM_FALLBACK_MODELS = ['nvidia/nemotron-3-super-120b-a12b:free', 'google/gemma-4-31b-it:free', 'openai/gpt-oss-20b:free'] +export const LLM_FALLBACK_MODELS = ['nvidia/nemotron-3-super-120b-a12b:free'] // How many repos make the digest, ranked by stars gained today. export const TRENDING_TOP_N = 8 +export const TELEGRAM_MAX_CHARS = 4096 +export const TRENDING_SUMMARY_MAX = 140 +export const TRENDING_TAGS_MAX = 5 +export const TRENDING_TAG_MAX = 24 export const COMPANY_CLEANUP_TABLE = 'mockTestUsers' export const COMPANY_CLEANUP_THRESHOLD_DAYS = '30' // ── AI news digest (Tavily) ────────────────────────────────────────────────── // How many stories make the digest. export const AI_NEWS_TOP_N = 4 -// Over-fetch: dedupe drops stories already sent, so asking for exactly 4 would -// deliver fewer than 4 on any day with repeats. export const AI_NEWS_FETCH_N = 10 export const AI_NEWS_LOOKBACK_DAYS = 1 export const AI_NEWS_SNIPPET_MAX = 200 -// Telegram drops the whole message past 4096 chars rather than truncating, and -// titleOf falls back to the full url when Tavily returns a blank title. export const AI_NEWS_TITLE_MAX = 200 -// Tavily's `score` is relevance to this query, not popularity — a vague query -// like "artificial intelligence" scores wellness blogs above model launches. -// Digest quality lives here and in AI_NEWS_DOMAINS, not in the ranking. -// -// Scope is AI *technology*: new models, developer tools, releases. Wording the -// query around releases and tools is what keeps funding rounds and stock moves -// out — asking for "industry announcements" pulled in ETFs and defence news. export const AI_NEWS_QUERY = 'new AI model releases, developer tools, and open source AI software' -// Tech press for launches and product news, developer sources for tooling and -// releases. Deliberately no reuters/bloomberg — their AI coverage is finance, -// which is what dragged in stock and ETF stories. -// -// ponytail: an allowlist, not a hard filter — Tavily treats include_domains as a -// strong hint, so the occasional off-list SEO listicle still slips through. Add -// exclude_domains only if one actually reaches a digest. export const AI_NEWS_DOMAINS = [ 'techcrunch.com', 'theverge.com', @@ -49,5 +33,5 @@ export const AI_NEWS_DOMAINS = [ export const DEFAULT_CRONJOB_TIME = '0 17 * * *' export const DEFAULT_CRONJOB_TIMEZONE = 'Australia/Sydney' // Change to Your local time zone when you need to use this constant value -export const NEWS_CRON_TIME = '0 8 * * *' // 8:00 AM Sydney -export const AI_NEWS_CRON_TIME = '30 8 * * *' // 8:30 AM Sydney — 30 min after the trending digest +export const NEWS_CRON_TIME = '0 7 * * *' // 7:00 AM Sydney +export const AI_NEWS_CRON_TIME = '30 7 * * *' // 7:30 AM Sydney — 30 min after the trending digest diff --git a/src/tools/news-telegram.test.ts b/src/tools/news-telegram.test.ts new file mode 100644 index 0000000..ed299ed --- /dev/null +++ b/src/tools/news-telegram.test.ts @@ -0,0 +1,97 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { buildTrendingMessage } from './news-telegram.tool.ts' +import { TELEGRAM_MAX_CHARS, TRENDING_SUMMARY_MAX, TRENDING_TAG_MAX, TRENDING_TAGS_MAX, TRENDING_TOP_N } from '../constants/index.ts' +import type { CuratedRepo } from '../schemas/index.ts' + +const repo = (over: Partial = {}): CuratedRepo => ({ + repo_name: 'facebook/react', + url: 'https://github.com/facebook/react', + description: 'A library for web and native user interfaces', + language: 'JavaScript', + stars: 238000, + today_stars: 412, + summary: 'The library that popularised the component model.', + tags: ['ui', 'framework'], + ...over, +}) + +test('numbers the repos and shows stars, language, tags and link', () => { + const message = buildTrendingMessage([repo(), repo({ repo_name: 'vuejs/core' })], '2026-08-01') + + assert.match(message, /1️⃣ facebook\/react<\/b>/) + assert.match(message, /2️⃣ vuejs\/core<\/b>/) + assert.match(message, /⭐ 238,000 \(\+412 today\) · JavaScript/) + assert.match(message, /🏷 #ui #framework/) + assert.match(message, /View on GitHub<\/a>/) +}) + +test('escapes html so a stray bracket cannot break Telegram parse_mode', () => { + const message = buildTrendingMessage([repo({ repo_name: 'a&c', summary: 'x < y && z' })], '2026-08-01') + + assert.match(message, /a<b>&c/) + assert.match(message, /x < y && z/) + assert.ok(!message.includes('&c')) +}) + +test('truncates an over-long summary at the bound', () => { + const message = buildTrendingMessage([repo({ summary: 'S'.repeat(TRENDING_SUMMARY_MAX + 200) })], '2026-08-01') + const italics = message.match(/(.*?)<\/i>/) + + assert.ok(italics) + assert.equal(italics[1].length, TRENDING_SUMMARY_MAX) +}) + +test('caps both the number of tags and the length of each one', () => { + const message = buildTrendingMessage( + [repo({ tags: Array.from({ length: TRENDING_TAGS_MAX + 20 }, () => 'T'.repeat(TRENDING_TAG_MAX + 30)) })], + '2026-08-01' + ) + const rendered = message.match(/🏷 (.*)/)?.[1] ?? '' + const tags = rendered.split(' ') + + assert.equal(tags.length, TRENDING_TAGS_MAX) + for (const t of tags) assert.ok(t.length <= TRENDING_TAG_MAX + 1, `tag too long: ${t.length}`) // +1 for the leading # +}) + +test('omits the tag line entirely when the curator returned none', () => { + const message = buildTrendingMessage([repo({ tags: [] })], '2026-08-01') + + assert.ok(!message.includes('🏷')) + assert.match(message, /⭐ 238,000/) // the rest of the entry is intact +}) + +// The bug this file was written for: tags were uncapped, so one verbose entry +// pushed the digest past Telegram's limit and Telegram rejected the WHOLE +// message — losing seven perfectly normal repos along with the one bad one. +test('one repo with a pathological tags array cannot cost the whole digest', () => { + const normal = Array.from({ length: TRENDING_TOP_N - 1 }, () => repo()) + const badApple = repo({ tags: Array.from({ length: 40 }, (_, i) => `machine-learning-infrastructure-tag-${i}`) }) + + const message = buildTrendingMessage([...normal, badApple], '2026-08-01') + + assert.ok(message.length <= TELEGRAM_MAX_CHARS, `message was ${message.length} chars`) + assert.match(message, /8️⃣/) // all eight still made it +}) + +test('a worst-case digest stays inside the limit Telegram hard-fails at', () => { + const worst = Array.from({ length: TRENDING_TOP_N }, () => + repo({ + repo_name: 'R'.repeat(140), + summary: 'S'.repeat(TRENDING_SUMMARY_MAX + 500), + tags: Array.from({ length: TRENDING_TAGS_MAX + 10 }, () => 'T'.repeat(TRENDING_TAG_MAX + 10)), + url: `https://github.com/${'u'.repeat(120)}`, + }) + ) + + assert.ok(buildTrendingMessage(worst, '2026-08-01').length <= TELEGRAM_MAX_CHARS) +}) + +// Escaping expands text after the per-field caps run (`&` becomes `&`), so +// the caps alone cannot guarantee the invariant — the trim guard does. +test('drops trailing repos rather than emitting a message Telegram will reject', () => { + const bomb = Array.from({ length: TRENDING_TOP_N }, () => repo({ summary: '&'.repeat(TRENDING_SUMMARY_MAX) })) + const message = buildTrendingMessage(bomb, '2026-08-01') + + assert.ok(message.length <= TELEGRAM_MAX_CHARS, `message was ${message.length} chars`) +}) diff --git a/src/tools/news-telegram.tool.ts b/src/tools/news-telegram.tool.ts index f0a9e47..0b43b35 100644 --- a/src/tools/news-telegram.tool.ts +++ b/src/tools/news-telegram.tool.ts @@ -1,7 +1,54 @@ import { DynamicStructuredTool } from '@langchain/core/tools' import { z } from 'zod' -import { escapeHtml } from '../agent/utils.js' +import { escapeHtml, truncate } from '../agent/utils.js' import { sendTelegramMessage } from './telegram.js' +import { TELEGRAM_MAX_CHARS, TRENDING_SUMMARY_MAX, TRENDING_TAG_MAX, TRENDING_TAGS_MAX } from '../constants/index.js' +import type { CuratedRepo } from '../schemas/index.js' + +const NUMBER_EMOJIS = ['1️⃣', '2️⃣', '3️⃣', '4️⃣', '5️⃣', '6️⃣', '7️⃣', '8️⃣'] + +function repoEntry(repo: CuratedRepo, index: number): string { + const num = NUMBER_EMOJIS[index] ?? `${index + 1}.` + const summary = truncate(repo.summary, TRENDING_SUMMARY_MAX) + const tags = repo.tags + .slice(0, TRENDING_TAGS_MAX) + .map((t) => `#${escapeHtml(truncate(t, TRENDING_TAG_MAX))}`) + .join(' ') + + const lines = [ + `${num} ${escapeHtml(repo.repo_name)}`, + `⭐ ${repo.stars.toLocaleString()} (+${repo.today_stars} today) · ${escapeHtml(repo.language)}`, + `${escapeHtml(summary)}`, + ] + + if (tags) lines.push(`🏷 ${tags}`) + + lines.push(`🔗 View on GitHub`) + + return lines.join('\n') +} + +function assemble(repos: CuratedRepo[], today: string): string { + return [ + `🔥 GitHub Trending — Daily Digest`, + `📅 ${escapeHtml(today)} · TypeScript / JavaScript`, + '', + '━━━━━━━━━━━━━━━━━━━━━━', + '', + repos.map(repoEntry).join('\n\n'), + '', + '━━━━━━━━━━━━━━━━━━━━━━', + `📊 ${repos.length} ${repos.length === 1 ? 'repo' : 'repos'} · Powered by GitHub Trending`, + ].join('\n') +} + +export function buildTrendingMessage(repos: CuratedRepo[], today: string): string { + let kept = repos + + while (kept.length > 1 && assemble(kept, today).length > TELEGRAM_MAX_CHARS) kept = kept.slice(0, -1) + + return assemble(kept, today) +} export const trendingTelegramTool = new DynamicStructuredTool({ name: 'send_trending_telegram', @@ -24,36 +71,7 @@ export const trendingTelegramTool = new DynamicStructuredTool({ }), func: async ({ repos }) => { const today = new Date().toISOString().split('T')[0] - - const numberEmojis = ['1️⃣', '2️⃣', '3️⃣', '4️⃣', '5️⃣', '6️⃣', '7️⃣', '8️⃣'] - - const repoLines = repos.map((r, i) => { - const num = numberEmojis[i] ?? `${i + 1}.` - // ponytail: the prompt asks for <140 chars, this is what actually guarantees it. - // Telegram hard-fails the whole message past 4096, so the bound lives in code. - const summary = r.summary.length > 140 ? `${r.summary.slice(0, 137)}...` : r.summary - return [ - `${num} ${escapeHtml(r.repo_name)}`, - `⭐ ${r.stars.toLocaleString()} (+${r.today_stars} today) · ${escapeHtml(r.language)}`, - `${escapeHtml(summary)}`, - `🏷 ${r.tags.map((t) => `#${escapeHtml(t)}`).join(' ')}`, - `🔗 View on GitHub`, - ].join('\n') - }) - - const message = [ - `🔥 GitHub Trending — Daily Digest`, - `📅 ${escapeHtml(today)} · TypeScript / JavaScript`, - '', - '━━━━━━━━━━━━━━━━━━━━━━', - '', - repoLines.join('\n\n'), - '', - '━━━━━━━━━━━━━━━━━━━━━━', - `📊 ${repos.length} repos · Powered by GitHub Trending`, - ].join('\n') - - const chatId = await sendTelegramMessage(message, 'Trending repos') + const chatId = await sendTelegramMessage(buildTrendingMessage(repos, today), 'Trending repos') return JSON.stringify({ success: true, chat_id: chatId, date: today }) },