feat: centralizza estrazione e raggruppamento link HTML in link_extractor + probe_html_portal

tests/test_link_extractor.py

@@ -0,0 +1,126 @@
+"""Tests per link_extractor — contratto pubblico: extract_data_links, group_links.
+
+pure_unit: extract_data_links, group_links
+"""
+
+import pytest
+from toolkit.scout.link_extractor import (
+    DataLink,
+    extract_data_links,
+    group_links,
+)
+
+
+class TestExtractDataLinks:
+    """pure_unit: estrazione link dati da HTML."""
+
+    @pytest.mark.pure_unit
+    def test_basic_formats(self):
+        """Estrae CSV, XLSX, JSON, ZIP, ignora PDF e HTML."""
+        html = """
+            <a href="/data.csv">CSV</a>
+            <a href="/data.xlsx">XLSX</a>
+            <a href="/data.json">JSON</a>
+            <a href="/data.zip">ZIP</a>
+            <a href="/about">HTML</a>
+            <a href="/report.pdf">PDF</a>
+        """
+        links = extract_data_links("https://ex.it", html)
+        assert len(links) == 4
+        assert {lnk.format for lnk in links} == {"CSV", "XLSX", "JSON", "ZIP"}
+
+    @pytest.mark.pure_unit
+    def test_skips_anchors_and_protocols(self):
+        """Ignora ancore, mailto, tel, javascript."""
+        html = """
+            <a href="#section">X</a>
+            <a href="mailto:a@b.it">X</a>
+            <a href="tel:+39">X</a>
+            <a href="javascript:void(0)">X</a>
+        """
+        assert extract_data_links("https://ex.it", html) == []
+
+    @pytest.mark.pure_unit
+    def test_resolves_relative_and_title(self):
+        """URL relativi risolti, aria-label come title."""
+        links = extract_data_links(
+            "https://ex.it/dir/",
+            '<a href="data.csv" aria-label="Report">x</a>',
+        )
+        assert links[0].url == "https://ex.it/dir/data.csv"
+        assert links[0].title == "Report"
+
+    @pytest.mark.pure_unit
+    def test_prefix_and_years(self):
+        """Prefisso e anni estratti dal filename."""
+        links = extract_data_links(
+            "https://ex.it",
+            '<a href="REG_tipo_reddito_2025.csv">x</a>',
+        )
+        assert links[0].prefix == "REG"
+        assert links[0].years == [2025]
+
+
+class TestGroupLinks:
+    """pure_unit: raggruppamento link per prefisso."""
+
+    @pytest.mark.pure_unit
+    def test_single_group(self):
+        """Stesso prefisso → un gruppo con anni e formati."""
+        links = [
+            DataLink(url="https://ex.it/REG_2024.csv", format="CSV", prefix="REG", years=[2024]),
+            DataLink(url="https://ex.it/REG_2025.csv", format="CSV", prefix="REG", years=[2025]),
+        ]
+        groups = group_links(links)
+        assert len(groups) == 1
+        assert groups[0].group_id == "REG"
+        assert groups[0].count == 2
+        assert groups[0].year_range == [2024, 2025]
+
+    @pytest.mark.pure_unit
+    def test_multiple_groups(self):
+        """Prefissi diversi → gruppi separati."""
+        links = [
+            DataLink(url="https://ex.it/REG_2024.csv", format="CSV", prefix="REG"),
+            DataLink(url="https://ex.it/CLA_2024.csv", format="CSV", prefix="CLA"),
+        ]
+        groups = group_links(links)
+        assert len(groups) == 2
+        assert {g.group_id for g in groups} == {"REG", "CLA"}
+
+    @pytest.mark.pure_unit
+    def test_no_prefix_becomes_other(self):
+        """Link senza prefisso → gruppo other."""
+        groups = group_links(
+            [
+                DataLink(url="https://ex.it/data.csv", format="CSV"),
+            ]
+        )
+        assert groups[0].group_id == "other"
+
+    @pytest.mark.pure_unit
+    def test_empty(self):
+        """Lista vuota → lista vuota."""
+        assert group_links([]) == []
+
+
+class TestMcpFormatsQueryString:
+    """pure_unit: MCP formats legacy usa path, non URL grezzo."""
+
+    @pytest.mark.pure_unit
+    def test_formats_from_path(self, monkeypatch):
+        """URL con query string → formato senza query."""
+        from toolkit.mcp.scout_ops import mcp_html_extract_links
+
+        html = """
+            <a href="https://ex.it/data.csv?download=1">CSV</a>
+            <a href="https://ex.it/archive.zip?t=123">ZIP</a>
+        """
+
+        def mock_fetch(url, **kw):
+            return {"html_text": html, "status_code": 200}
+
+        monkeypatch.setattr("toolkit.mcp.scout_ops.fetch_html_body", mock_fetch)
+        result = mcp_html_extract_links("https://ex.it/")
+        assert result["formats"] == {"csv": 1, "zip": 1}
+        assert result["data_links"][0]["format"] == "CSV"

tests/test_probe_sitemap.py

@@ -0,0 +1,126 @@
+"""Tests per probe — fetch_sitemap_pages con sitemap index.
+
+pure_unit: fetch_sitemap_pages parsing XML (mock HTTP)
+"""
+
+from unittest.mock import patch, MagicMock
+
+import pytest
+from toolkit.scout.probe import fetch_sitemap_pages
+
+
+# ---------------------------------------------------------------------------
+# pure_unit — fetch_sitemap_pages
+# ---------------------------------------------------------------------------
+
+
+class TestFetchSitemapPages:
+    """pure_unit: parsing sitemap XML."""
+
+    @pytest.mark.pure_unit
+    def test_standard_sitemap(self):
+        """Sitemap standard <urlset><url><loc>."""
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://ex.it/page1</loc></url>
+  <url><loc>https://ex.it/page2</loc></url>
+</urlset>"""
+        pages = self._mock_fetch(xml)
+        assert pages == ["https://ex.it/page1", "https://ex.it/page2"]
+
+    @pytest.mark.pure_unit
+    def test_sitemap_index(self):
+        """Sitemap index <sitemapindex><sitemap><loc> con fetch ricorsivo."""
+        index_xml = """<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>https://ex.it/sub1.xml</loc></sitemap>
+  <sitemap><loc>https://ex.it/sub2.xml</loc></sitemap>
+</sitemapindex>"""
+        sub1_xml = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://ex.it/page1</loc></url>
+</urlset>"""
+        sub2_xml = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://ex.it/page2</loc></url>
+  <url><loc>https://ex.it/page3</loc></url>
+</urlset>"""
+
+        pages = self._mock_fetch_index(
+            index_xml,
+            {
+                "https://ex.it/sub1.xml": sub1_xml,
+                "https://ex.it/sub2.xml": sub2_xml,
+            },
+        )
+        assert pages == ["https://ex.it/page1", "https://ex.it/page2", "https://ex.it/page3"]
+
+    @pytest.mark.pure_unit
+    def test_sitemap_index_no_namespace(self):
+        """Sitemap index senza namespace XML."""
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex>
+  <sitemap><loc>https://ex.it/sub.xml</loc></sitemap>
+</sitemapindex>"""
+        sub_xml = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset>
+  <url><loc>https://ex.it/page</loc></url>
+</urlset>"""
+        pages = self._mock_fetch_index(xml, {"https://ex.it/sub.xml": sub_xml})
+        assert pages == ["https://ex.it/page"]
+
+    @pytest.mark.pure_unit
+    def test_unreachable_sitemap(self):
+        """Sitemap non raggiungibile → lista vuota."""
+        with patch("toolkit.scout.probe.HttpClient") as MockClient:
+            instance = MockClient.return_value
+            instance.get.return_value = MagicMock(is_ok=False)
+            pages = fetch_sitemap_pages("https://ex.it/sitemap.xml", timeout=5)
+            assert pages == []
+
+    @pytest.mark.pure_unit
+    def test_malformed_xml(self):
+        """XML malformato → lista vuota."""
+        with patch("toolkit.scout.probe.HttpClient") as MockClient:
+            instance = MockClient.return_value
+            mock_resp = MagicMock()
+            mock_resp.status_code = 200
+            mock_resp.text = "not xml"
+            instance.get.return_value = MagicMock(
+                is_ok=True, response=mock_resp, err=None, ssl_fallback_used=False
+            )
+            pages = fetch_sitemap_pages("https://ex.it/sitemap.xml", timeout=5)
+            assert pages == []
+
+    # -- helpers --
+
+    def _mock_fetch(self, xml: str) -> list[str]:
+        """Mock HttpClient per sitemap standard."""
+        with patch("toolkit.scout.probe.HttpClient") as MockClient:
+            instance = MockClient.return_value
+            mock_resp = MagicMock()
+            mock_resp.status_code = 200
+            mock_resp.text = xml
+            instance.get.return_value = MagicMock(
+                is_ok=True, response=mock_resp, err=None, ssl_fallback_used=False
+            )
+            return fetch_sitemap_pages("https://ex.it/sitemap.xml", timeout=5)
+
+    def _mock_fetch_index(self, index_xml: str, subs: dict[str, str]) -> list[str]:
+        """Mock HttpClient per sitemap index con sotto-sitemap."""
+
+        def mock_get(url):
+            mock_resp = MagicMock()
+            mock_resp.status_code = 200
+            if url == "https://ex.it/sitemap.xml":
+                mock_resp.text = index_xml
+            elif url in subs:
+                mock_resp.text = subs[url]
+            else:
+                mock_resp.text = ""
+            return MagicMock(is_ok=True, response=mock_resp, err=None, ssl_fallback_used=False)
+
+        with patch("toolkit.scout.probe.HttpClient") as MockClient:
+            instance = MockClient.return_value
+            instance.get.side_effect = mock_get
+            return fetch_sitemap_pages("https://ex.it/sitemap.xml", timeout=5)

toolkit/mcp/scout_ops.py

@@ -10,14 +10,17 @@
 
 from toolkit.plugins.sparql import SparqlSource
 from toolkit.scout.http import (
-    extract_candidate_links,
     fetch_ckan_package,
     fetch_html_body,
     list_sdmx_dataflows as _list_sdmx_dataflows,
     probe_url_headers,
     search_ckan_datasets as _search_ckan_datasets,
 )
 from toolkit.scout.infer import infer_topics
+from toolkit.scout.link_extractor import (
+    extract_data_links,
+    group_links,
+)
 from toolkit.scout.probe import probe_url, probe_url_routed
 
 
@@ -144,11 +147,18 @@ def mcp_ckan_package_show(
 def mcp_html_extract_links(url: str, timeout: int = 20) -> dict[str, Any]:
     """Estrae link a file dati (CSV, JSON, XLSX, ZIP, XML) da una pagina HTML.
 
-    1. Scrive ``fetch_html_body()`` per scaricare la pagina.
-    2. ``extract_candidate_links()`` per estrarre i link ai dati.
+    1. Scarica la pagina via ``fetch_html_body()``.
+    2. Usa ``extract_data_links()`` per estrarre link con metadati (formato,
+       prefisso, anni).
+    3. Usa ``group_links()`` per raggruppare i link in dataset.
 
     Returns:
-        Dict con url, total, formats, links, is_reachable.
+        Dict con:
+        - ``url``, ``is_reachable``, ``http_status``
+        - ``links`` (list[str], backward compat) e ``total``
+        - ``formats`` (dict[str, int], backward compat, lowercase estensione)
+        - ``data_links`` (list[dict] con url, format, title, prefix, years, page_url)
+        - ``groups`` (list[dict] con group_id, prefix, count, year_range, formats)
     """
     try:
         body = fetch_html_body(url, timeout=timeout)
@@ -160,14 +170,24 @@ def mcp_html_extract_links(url: str, timeout: int = 20) -> dict[str, Any]:
             "links": [],
             "total": 0,
             "formats": {},
+            "data_links": [],
+            "groups": [],
         }
 
     html_text = body.get("html_text", "")
-    links = extract_candidate_links(url, html_text)
+    data_links = extract_data_links(url, html_text)
+    groups = group_links(data_links)
+
+    # Backward compat: lista di URL semplici
+    links = [dl.url for dl in data_links]
+
+    # Backward compat: conteggio per estensione (lowercase, dal path senza query)
+    from urllib.parse import urlparse
 
     formats: dict[str, int] = {}
-    for link in links:
-        ext = link.rsplit(".", 1)[-1].lower() if "." in link else "unknown"
+    for dl in data_links:
+        path = urlparse(dl.url).path
+        ext = path.rsplit(".", 1)[-1].lower() if "." in path else "unknown"
         formats[ext] = formats.get(ext, 0) + 1
 
     return {
@@ -177,6 +197,28 @@ def mcp_html_extract_links(url: str, timeout: int = 20) -> dict[str, Any]:
         "total": len(links),
         "links": links,
         "formats": formats,
+        "data_links": [
+            {
+                "url": dl.url,
+                "format": dl.format,
+                "title": dl.title,
+                "prefix": dl.prefix,
+                "years": dl.years,
+                "page_url": dl.page_url or url,
+            }
+            for dl in data_links
+        ],
+        "groups": [
+            {
+                "group_id": g.group_id,
+                "prefix": g.prefix,
+                "count": g.count,
+                "year_range": g.year_range,
+                "formats": sorted(g.formats),
+                "title": g.title,
+            }
+            for g in groups
+        ],
     }
 
 

toolkit/scout/__init__.py

@@ -7,10 +7,11 @@
 - Notebook e script interni
 
 Moduli:
-  http      → HTTP transport: probe, fetch, format detection, CKAN/SDMX
-  sparql    → SPARQL scout: named graph discovery, schema inference
-  infer     → Inferenze pure: anni, granularità, topic
-  probe     → Orchestrazione: probe_url(), probe_url_routed()
+  http           → HTTP transport: probe, fetch, format detection, CKAN/SDMX
+  link_extractor → Estrazione e raggruppamento link dati da HTML
+  sparql         → SPARQL scout: named graph discovery, schema inference
+  infer          → Inferenze pure: anni, granularità, topic
+  probe          → Orchestrazione: probe_url(), probe_url_routed()
 
 Scaffold non e' piu' in scout. Usa toolkit.scaffold.full e toolkit.scaffold.sources.
 """
@@ -34,13 +35,20 @@
     resolve_preview_kind,
     detect_ckan_in_html,
     extract_ckan_dataset_id,
-    extract_candidate_links,
     DEFAULT_TIMEOUT,
     DEFAULT_USER_AGENT,
     CANDIDATE_EXTENSIONS,
     EXTENDED_EXTENSIONS,
 )
 
+from toolkit.scout.link_extractor import (  # noqa: F401
+    DataLink,
+    LinkGroup,
+    extract_data_links,
+    group_links,
+    extract_candidate_links,
+)
+
 from toolkit.scout.infer import (  # noqa: F401
     infer_years,
     suggest_years,
@@ -50,6 +58,9 @@
 )
 
 from toolkit.scout.probe import (  # noqa: F401
+    PortalProfile,
+    fetch_sitemap_pages,
+    probe_html_portal,
     probe_url,
     probe_url_routed,
 )