diff --git a/src/core/seo_extractor.py b/src/core/seo_extractor.py index 08e2e2e..13f0b98 100644 --- a/src/core/seo_extractor.py +++ b/src/core/seo_extractor.py @@ -1,9 +1,32 @@ """SEO data extraction from HTML content""" +import copy import re import json from urllib.parse import urljoin, urlparse +CHROME_TAGS = ('script', 'style', 'noscript', 'template', + 'nav', 'header', 'footer', 'aside') +_WORD_RE = re.compile(r'\b\w+\b') + + +def _count_content_words(soup): + """Words in the page body, without scripts, styles or repeated chrome. + + Works on a copy: the caller keeps using this soup for links, headings and + images, so the elements removed here must not disappear from it. + """ + try: + body = copy.copy(soup) + except Exception: + body = soup + for tag in body(CHROME_TAGS): + tag.decompose() + main = body.find('main') or body.find('body') or body + return len(_WORD_RE.findall(main.get_text(' ', strip=True))) + + + class SEOExtractor: """Extracts SEO-related data from HTML content""" @@ -28,10 +51,16 @@ def extract_basic_seo_data(soup, result): h3_tags = soup.find_all('h3') result['h3'] = [h3.get_text().strip() for h3 in h3_tags[:10]] - # Count words - text_content = soup.get_text() - words = re.findall(r'\b\w+\b', text_content) - result['word_count'] = len(words) + # Count words in the page's own content. + # + # soup.get_text() on the whole document counted the contents of + #