diff --git a/src/tools/trending-scrape.test.ts b/src/tools/trending-scrape.test.ts
new file mode 100644
index 0000000..e36e41e
--- /dev/null
+++ b/src/tools/trending-scrape.test.ts
@@ -0,0 +1,56 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { parseTrendingHtml } from './trending-scrape.tool.ts'
+
+// Trimmed from a real github.com/trending/typescript?since=daily response.
+const article = (description: string) => `
+
+
+
+ ${description}
+
+
+
TypeScript
+
+ 386
+
+
+ 28 stars today
+
+`
+
+const descriptionOf = (raw: string) => parseTrendingHtml(article(raw))[0].description
+
+test('decodes the escaped ampersand GitHub emits in repo descriptions', () => {
+ assert.equal(descriptionOf('Open alternative to Kommo, Octadesk & Intercom'), 'Open alternative to Kommo, Octadesk & Intercom')
+})
+
+test('decodes the rest of the entities a description can carry', () => {
+ assert.equal(descriptionOf('Type-safe <div> helpers'), 'Type-safe
helpers')
+ assert.equal(descriptionOf('The "batteries included" runtime'), 'The "batteries included" runtime')
+ assert.equal(descriptionOf('Rust's borrow checker, in TS'), "Rust's borrow checker, in TS")
+ assert.equal(descriptionOf('Ship it 🚀 fast'), 'Ship it 🚀 fast')
+})
+
+test('a decoded angle bracket is text, not markup to strip a second time', () => {
+ assert.equal(descriptionOf('Render <Suspense> on the server'), 'Render
on the server')
+})
+
+test('leaves something that only looks like an entity untouched', () => {
+ assert.equal(descriptionOf('Sold ¬anentity; separately'), 'Sold ¬anentity; separately')
+ assert.equal(descriptionOf('Costs AT&T money'), 'Costs AT&T money')
+ assert.equal(descriptionOf('Out of range here'), 'Out of range here')
+})
+
+test('still reads the name, language and both star counts', () => {
+ const [repo] = parseTrendingHtml(article('A CRM'))
+
+ assert.equal(repo.name, 'melgarafael/DeskcommCRM')
+ assert.equal(repo.url, 'https://github.com/melgarafael/DeskcommCRM')
+ assert.equal(repo.language, 'TypeScript')
+ assert.equal(repo.stars, 386)
+ assert.equal(repo.todayStars, 28)
+})
diff --git a/src/tools/trending-scrape.tool.ts b/src/tools/trending-scrape.tool.ts
index e8ac97f..201f79e 100644
--- a/src/tools/trending-scrape.tool.ts
+++ b/src/tools/trending-scrape.tool.ts
@@ -3,7 +3,23 @@ import { DynamicStructuredTool } from '@langchain/core/tools'
import { z } from 'zod'
import { TrendingRepo } from '../schemas/index.ts'
-function parseTrendingHtml(html: string): TrendingRepo[] {
+const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
+
+function decodeNumericRef(ref: string): string | null {
+ const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10)
+
+ return Number.isNaN(code) || code > 0x10ffff ? null : String.fromCodePoint(code)
+}
+
+function decodeEntities(text: string): string {
+ return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, ref: string) => {
+ const decoded = ref[0] === '#' ? decodeNumericRef(ref) : NAMED_ENTITIES[ref.toLowerCase()]
+
+ return decoded ?? entity
+ })
+}
+
+export function parseTrendingHtml(html: string): TrendingRepo[] {
const repos: TrendingRepo[] = []
const articleRegex = /([\s\S]*?)<\/article>/g
@@ -18,7 +34,7 @@ function parseTrendingHtml(html: string): TrendingRepo[] {
// Description
const descMatch = block.match(/]*>([\s\S]*?)<\/p>/)
- const description = descMatch ? descMatch[1].replace(/<[^>]+>/g, '').trim() : ''
+ const description = descMatch ? decodeEntities(descMatch[1].replace(/<[^>]+>/g, '')).trim() : ''
// Language
const langMatch = block.match(/([\s\S]*?)<\/span>/)