fix: decode HTML entities in scraped trending descriptions

src/tools/trending-scrape.test.ts

@@ -0,0 +1,56 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { parseTrendingHtml } from './trending-scrape.tool.ts'
+
+// Trimmed from a real github.com/trending/typescript?since=daily response.
+const article = (description: string) => `
+<article class="Box-row">
+  <h2 class="h3 lh-condensed">
+    <a href="/melgarafael/DeskcommCRM" data-view-component="true" class="Link"><svg class="octicon"></svg>
+    <span data-view-component="true" class="text-normal">melgarafael /</span>
+    DeskcommCRM</a>  </h2>
+  <p class="col-9 color-fg-muted my-1 tmp-pr-4">
+    ${description}
+  </p>
+  <div class="f6 color-fg-muted mt-2">
+    <span itemprop="programmingLanguage">TypeScript</span>
+    <a href="/melgarafael/DeskcommCRM/stargazers" class="Link"><svg class="octicon"></svg>
+    386</a>
+    <span data-view-component="true" class="d-inline-block float-sm-right">
+      <svg class="octicon"></svg>
+      28 stars today
+    </span>  </div>
+</article>`
+
+const descriptionOf = (raw: string) => parseTrendingHtml(article(raw))[0].description
+
+test('decodes the escaped ampersand GitHub emits in repo descriptions', () => {
+  assert.equal(descriptionOf('Open alternative to Kommo, Octadesk &amp; Intercom'), 'Open alternative to Kommo, Octadesk & Intercom')
+})
+
+test('decodes the rest of the entities a description can carry', () => {
+  assert.equal(descriptionOf('Type-safe &lt;div&gt; helpers'), 'Type-safe <div> helpers')
+  assert.equal(descriptionOf('The &quot;batteries included&quot; runtime'), 'The "batteries included" runtime')
+  assert.equal(descriptionOf('Rust&#39;s borrow checker, in TS'), "Rust's borrow checker, in TS")
+  assert.equal(descriptionOf('Ship it &#x1F680; fast'), 'Ship it 🚀 fast')
+})
+
+test('a decoded angle bracket is text, not markup to strip a second time', () => {
+  assert.equal(descriptionOf('Render &lt;Suspense&gt; on the server'), 'Render <Suspense> on the server')
+})
+
+test('leaves something that only looks like an entity untouched', () => {
+  assert.equal(descriptionOf('Sold &notanentity; separately'), 'Sold &notanentity; separately')
+  assert.equal(descriptionOf('Costs AT&T money'), 'Costs AT&T money')
+  assert.equal(descriptionOf('Out of range &#9999999; here'), 'Out of range &#9999999; here')
+})
+
+test('still reads the name, language and both star counts', () => {
+  const [repo] = parseTrendingHtml(article('A CRM'))
+
+  assert.equal(repo.name, 'melgarafael/DeskcommCRM')
+  assert.equal(repo.url, 'https://github.com/melgarafael/DeskcommCRM')
+  assert.equal(repo.language, 'TypeScript')
+  assert.equal(repo.stars, 386)
+  assert.equal(repo.todayStars, 28)
+})

src/tools/trending-scrape.tool.ts

@@ -3,7 +3,23 @@ import { DynamicStructuredTool } from '@langchain/core/tools'
 import { z } from 'zod'
 import { TrendingRepo } from '../schemas/index.ts'
 
-function parseTrendingHtml(html: string): TrendingRepo[] {
+const NAMED_ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
+
+function decodeNumericRef(ref: string): string | null {
+  const code = ref[1].toLowerCase() === 'x' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10)
+
+  return Number.isNaN(code) || code > 0x10ffff ? null : String.fromCodePoint(code)
+}
+
+function decodeEntities(text: string): string {
+  return text.replace(/&(#\d+|#x[0-9a-f]+|[a-z]+);/gi, (entity, ref: string) => {
+    const decoded = ref[0] === '#' ? decodeNumericRef(ref) : NAMED_ENTITIES[ref.toLowerCase()]
+
+    return decoded ?? entity
+  })
+}
+
+export function parseTrendingHtml(html: string): TrendingRepo[] {
   const repos: TrendingRepo[] = []
   const articleRegex = /<article class="Box-row">([\s\S]*?)<\/article>/g
 
@@ -18,7 +34,7 @@ function parseTrendingHtml(html: string): TrendingRepo[] {
 
     // Description
     const descMatch = block.match(/<p class="col-9[^"]*"[^>]*>([\s\S]*?)<\/p>/)
-    const description = descMatch ? descMatch[1].replace(/<[^>]+>/g, '').trim() : ''
+    const description = descMatch ? decodeEntities(descMatch[1].replace(/<[^>]+>/g, '')).trim() : ''
 
     // Language
     const langMatch = block.match(/<span itemprop="programmingLanguage">([\s\S]*?)<\/span>/)