From c469ac2ed3053145140937833c19280520812da9 Mon Sep 17 00:00:00 2001
From: amedipiran <mcc@cyntora.se>
Date: Thu, 17 Sep 2026 11:25:08 +0200
Subject: [PATCH] Exclude scripts, styles and chrome from word_count

soup.get_text() on the whole document counted the contents of <script> and
<style> as words, along with the navigation, header and footer that repeat
on every page. On a site with a large menu that is over a thousand words of
chrome on every page.

Measured on one WooCommerce product page: 1333 words before, 559 after.

This quietly broke everything built on word_count. The low-content check
fires below 300 words and never fired at all, and word_count carries 0.10
weight in near-duplicate detection.

Counts on a copy so the caller's soup keeps its links, headings and images.
---
 src/core/seo_extractor.py | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/core/seo_extractor.py b/src/core/seo_extractor.py
index 08e2e2e..13f0b98 100644
--- a/src/core/seo_extractor.py
+++ b/src/core/seo_extractor.py
@@ -1,9 +1,32 @@
 """SEO data extraction from HTML content"""
+import copy
 import re
 import json
 from urllib.parse import urljoin, urlparse
 
 
+CHROME_TAGS = ('script', 'style', 'noscript', 'template',
+               'nav', 'header', 'footer', 'aside')
+_WORD_RE = re.compile(r'\b\w+\b')
+
+
+def _count_content_words(soup):
+    """Words in the page body, without scripts, styles or repeated chrome.
+
+    Works on a copy: the caller keeps using this soup for links, headings and
+    images, so the elements removed here must not disappear from it.
+    """
+    try:
+        body = copy.copy(soup)
+    except Exception:
+        body = soup
+    for tag in body(CHROME_TAGS):
+        tag.decompose()
+    main = body.find('main') or body.find('body') or body
+    return len(_WORD_RE.findall(main.get_text(' ', strip=True)))
+
+
+
 class SEOExtractor:
     """Extracts SEO-related data from HTML content"""
 
@@ -28,10 +51,16 @@ def extract_basic_seo_data(soup, result):
         h3_tags = soup.find_all('h3')
         result['h3'] = [h3.get_text().strip() for h3 in h3_tags[:10]]
 
-        # Count words
-        text_content = soup.get_text()
-        words = re.findall(r'\b\w+\b', text_content)
-        result['word_count'] = len(words)
+        # Count words in the page's own content.
+        #
+        # soup.get_text() on the whole document counted the contents of
+        # <script> and <style> as words, plus the navigation, header and
+        # footer that repeat on every page. On a site with a large menu that
+        # is over a thousand words of chrome added to every page, which makes
+        # word_count meaningless and quietly breaks everything built on it:
+        # the low-content check below 300 words never fires, and word_count
+        # carries weight in near-duplicate detection.
+        result['word_count'] = _count_content_words(soup)
 
         # Extract language
         html_tag = soup.find('html')