From 140018eeee60717c9e2606e90b091fe5b9763615 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 10:18:49 +0000 Subject: [PATCH 1/2] fix: decode HTML entities in scraped trending descriptions Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013v6SNv4axPxsCPcmEFNk86 --- src/tools/trending-scrape.test.ts | 55 +++++++++++++++++++++++++++++++ src/tools/trending-scrape.tool.ts | 16 +++++++-- 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 src/tools/trending-scrape.test.ts diff --git a/src/tools/trending-scrape.test.ts b/src/tools/trending-scrape.test.ts new file mode 100644 index 0000000..8d8db78 --- /dev/null +++ b/src/tools/trending-scrape.test.ts @@ -0,0 +1,55 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { parseTrendingHtml } from './trending-scrape.tool.ts' + +// Trimmed from a real github.com/trending/typescript?since=daily response. +const article = (description: string) => ` +` + +const descriptionOf = (raw: string) => parseTrendingHtml(article(raw))[0].description + +test('decodes the escaped ampersand GitHub emits in repo descriptions', () => { + assert.equal(descriptionOf('Open alternative to Kommo, Octadesk & Intercom'), 'Open alternative to Kommo, Octadesk & Intercom') +}) + +test('decodes the rest of the entities a description can carry', () => { + assert.equal(descriptionOf('Type-safe <div> helpers'), 'Type-safe
helpers') + assert.equal(descriptionOf('The "batteries included" runtime'), 'The "batteries included" runtime') + assert.equal(descriptionOf('Rust's borrow checker, in TS'), "Rust's borrow checker, in TS") + assert.equal(descriptionOf('Ship it 🚀 fast'), 'Ship it 🚀 fast') +}) + +test('a decoded angle bracket is text, not markup to strip a second time', () => { + assert.equal(descriptionOf('Render <Suspense> on the server'), 'Render on the server') +}) + +test('leaves something that only looks like an entity untouched', () => { + assert.equal(descriptionOf('Sold ¬anentity; separately'), 'Sold ¬anentity; separately') + assert.equal(descriptionOf('Costs AT&T money'), 'Costs AT&T money') +}) + +test('still reads the name, language and both star counts', () => { + const [repo] = parseTrendingHtml(article('A CRM')) + + assert.equal(repo.name, 'melgarafael/DeskcommCRM') + assert.equal(repo.url, 'https://github.com/melgarafael/DeskcommCRM') + assert.equal(repo.language, 'TypeScript') + assert.equal(repo.stars, 386) + assert.equal(repo.todayStars, 28) +}) diff --git a/src/tools/trending-scrape.tool.ts b/src/tools/trending-scrape.tool.ts index e8ac97f..2add336 100644 --- a/src/tools/trending-scrape.tool.ts +++ b/src/tools/trending-scrape.tool.ts @@ -3,7 +3,19 @@ import { DynamicStructuredTool } from '@langchain/core/tools' import { z } from 'zod' import { TrendingRepo } from '../schemas/index.ts' -function parseTrendingHtml(html: string): TrendingRepo[] { +const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' } + +function decodeEntities(text: string): string { + return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, ref: string) => { + if (ref[0] !== '#') return NAMED_ENTITIES[ref.toLowerCase()] ?? entity + + const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10) + + return Number.isNaN(code) || code > 0x10ffff ? entity : String.fromCodePoint(code) + }) +} + +export function parseTrendingHtml(html: string): TrendingRepo[] { const repos: TrendingRepo[] = [] const articleRegex = /
([\s\S]*?)<\/article>/g @@ -18,7 +30,7 @@ function parseTrendingHtml(html: string): TrendingRepo[] { // Description const descMatch = block.match(/

]*>([\s\S]*?)<\/p>/) - const description = descMatch ? descMatch[1].replace(/<[^>]+>/g, '').trim() : '' + const description = descMatch ? decodeEntities(descMatch[1].replace(/<[^>]+>/g, '')).trim() : '' // Language const langMatch = block.match(/([\s\S]*?)<\/span>/) From d11ead4345347b4cf1e1a11b146319989d0c4056 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 10:22:07 +0000 Subject: [PATCH 2/2] refactor: split numeric entity decoding into its own function Keeps the replace callback's CRAP score under Fallow's threshold and covers the out-of-range code point guard. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013v6SNv4axPxsCPcmEFNk86 --- src/tools/trending-scrape.test.ts | 1 + src/tools/trending-scrape.tool.ts | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tools/trending-scrape.test.ts b/src/tools/trending-scrape.test.ts index 8d8db78..e36e41e 100644 --- a/src/tools/trending-scrape.test.ts +++ b/src/tools/trending-scrape.test.ts @@ -42,6 +42,7 @@ test('a decoded angle bracket is text, not markup to strip a second time', () => test('leaves something that only looks like an entity untouched', () => { assert.equal(descriptionOf('Sold ¬anentity; separately'), 'Sold ¬anentity; separately') assert.equal(descriptionOf('Costs AT&T money'), 'Costs AT&T money') + assert.equal(descriptionOf('Out of range � here'), 'Out of range � here') }) test('still reads the name, language and both star counts', () => { diff --git a/src/tools/trending-scrape.tool.ts b/src/tools/trending-scrape.tool.ts index 2add336..201f79e 100644 --- a/src/tools/trending-scrape.tool.ts +++ b/src/tools/trending-scrape.tool.ts @@ -5,13 +5,17 @@ import { TrendingRepo } from '../schemas/index.ts' const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' } +function decodeNumericRef(ref: string): string | null { + const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10) + + return Number.isNaN(code) || code > 0x10ffff ? null : String.fromCodePoint(code) +} + function decodeEntities(text: string): string { return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, ref: string) => { - if (ref[0] !== '#') return NAMED_ENTITIES[ref.toLowerCase()] ?? entity - - const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10) + const decoded = ref[0] === '#' ? decodeNumericRef(ref) : NAMED_ENTITIES[ref.toLowerCase()] - return Number.isNaN(code) || code > 0x10ffff ? entity : String.fromCodePoint(code) + return decoded ?? entity }) }