diff --git a/src/tools/trending-scrape.test.ts b/src/tools/trending-scrape.test.ts new file mode 100644 index 0000000..e36e41e --- /dev/null +++ b/src/tools/trending-scrape.test.ts @@ -0,0 +1,56 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { parseTrendingHtml } from './trending-scrape.tool.ts' + +// Trimmed from a real github.com/trending/typescript?since=daily response. +const article = (description: string) => ` +
+

+ + melgarafael / + DeskcommCRM

+

+ ${description} +

+
+ TypeScript + + 386 + + + 28 stars today +
+
` + +const descriptionOf = (raw: string) => parseTrendingHtml(article(raw))[0].description + +test('decodes the escaped ampersand GitHub emits in repo descriptions', () => { + assert.equal(descriptionOf('Open alternative to Kommo, Octadesk & Intercom'), 'Open alternative to Kommo, Octadesk & Intercom') +}) + +test('decodes the rest of the entities a description can carry', () => { + assert.equal(descriptionOf('Type-safe <div> helpers'), 'Type-safe
helpers') + assert.equal(descriptionOf('The "batteries included" runtime'), 'The "batteries included" runtime') + assert.equal(descriptionOf('Rust's borrow checker, in TS'), "Rust's borrow checker, in TS") + assert.equal(descriptionOf('Ship it 🚀 fast'), 'Ship it 🚀 fast') +}) + +test('a decoded angle bracket is text, not markup to strip a second time', () => { + assert.equal(descriptionOf('Render <Suspense> on the server'), 'Render on the server') +}) + +test('leaves something that only looks like an entity untouched', () => { + assert.equal(descriptionOf('Sold ¬anentity; separately'), 'Sold ¬anentity; separately') + assert.equal(descriptionOf('Costs AT&T money'), 'Costs AT&T money') + assert.equal(descriptionOf('Out of range � here'), 'Out of range � here') +}) + +test('still reads the name, language and both star counts', () => { + const [repo] = parseTrendingHtml(article('A CRM')) + + assert.equal(repo.name, 'melgarafael/DeskcommCRM') + assert.equal(repo.url, 'https://github.com/melgarafael/DeskcommCRM') + assert.equal(repo.language, 'TypeScript') + assert.equal(repo.stars, 386) + assert.equal(repo.todayStars, 28) +}) diff --git a/src/tools/trending-scrape.tool.ts b/src/tools/trending-scrape.tool.ts index e8ac97f..201f79e 100644 --- a/src/tools/trending-scrape.tool.ts +++ b/src/tools/trending-scrape.tool.ts @@ -3,7 +3,23 @@ import { DynamicStructuredTool } from '@langchain/core/tools' import { z } from 'zod' import { TrendingRepo } from '../schemas/index.ts' -function parseTrendingHtml(html: string): TrendingRepo[] { +const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' } + +function decodeNumericRef(ref: string): string | null { + const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10) + + return Number.isNaN(code) || code > 0x10ffff ? null : String.fromCodePoint(code) +} + +function decodeEntities(text: string): string { + return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, ref: string) => { + const decoded = ref[0] === '#' ? decodeNumericRef(ref) : NAMED_ENTITIES[ref.toLowerCase()] + + return decoded ?? entity + }) +} + +export function parseTrendingHtml(html: string): TrendingRepo[] { const repos: TrendingRepo[] = [] const articleRegex = /
([\s\S]*?)<\/article>/g @@ -18,7 +34,7 @@ function parseTrendingHtml(html: string): TrendingRepo[] { // Description const descMatch = block.match(/

]*>([\s\S]*?)<\/p>/) - const description = descMatch ? descMatch[1].replace(/<[^>]+>/g, '').trim() : '' + const description = descMatch ? decodeEntities(descMatch[1].replace(/<[^>]+>/g, '')).trim() : '' // Language const langMatch = block.match(/([\s\S]*?)<\/span>/)