Add Drugs.com scraper and tests

datasets/amfv_datasets/scraping/__init__.py

@@ -9,6 +9,16 @@
     scrape_listing_documents,
 )
 from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
+from amfv_datasets.scraping.drugs_com import (
+    DrugsComFetchError,
+    DrugsComPageRef,
+    build_drugs_com_article_text,
+    drugscom_ref_from_url,
+    list_drug_refs,
+    list_two_letter_pages,
+    scrape_drugs_com,
+    scrape_drugs_com_page,
+)
 from amfv_datasets.scraping.html import (
     LinkMode,
     absolute_unique_urls,
@@ -29,6 +39,9 @@
 )
 
 __all__ = [
+    "DRUGSCOM_BASE_URL",
+    "DrugsComFetchError",
+    "DrugsComPageRef",
     "GuidanceRef",
     "GuidanceListingPage",
     "LinkMode",
@@ -40,14 +53,20 @@
     "ScraperSource",
     "USER_AGENT",
     "absolute_unique_urls",
+    "build_drugs_com_article_text",
     "build_guideline_text",
     "clean_text",
-    "document_title",
     "default_client",
+    "document_title",
+    "drugscom_ref_from_url",
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
+    "list_drug_refs",
     "list_published_guidance",
+    "list_two_letter_pages",
+    "scrape_drugs_com",
+    "scrape_drugs_com_page",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",

datasets/amfv_datasets/scraping/base.py

@@ -74,7 +74,10 @@ def scrape_listing_documents[ListingItemT](
     documents: int | None,
     client_factory: Callable[[], AbstractContextManager[httpx.Client]],
     list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument],
+    scrape_item: Callable[
+        [httpx.Client, ListingItemT],
+        ScrapedDocument | None,
+    ],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -99,6 +102,7 @@ def scrape_listing_documents[ListingItemT](
     with client_factory() as client:
         page = 1
         scraped = 0
+        attempted = 0
         page_items = list(first_page_items) if first_page_items is not None else None
         while documents is None or scraped < documents:
             if page_items is None:
@@ -111,9 +115,16 @@ def scrape_listing_documents[ListingItemT](
             for item in items:
                 if documents is not None and scraped >= documents:
                     break
-                if scraped and document_delay_seconds:
+                if attempted and document_delay_seconds:
                     time.sleep(document_delay_seconds)
-                yield scrape_item(client, item)
+
+                document = scrape_item(client, item)
+                attempted += 1
+
+                if document is None:
+                    continue
+
+                yield document
                 scraped += 1
             page += 1
 

datasets/amfv_datasets/scraping/cli.py

@@ -8,6 +8,7 @@
 from collections.abc import Iterable
 from dataclasses import asdict
 from enum import StrEnum
+from itertools import chain
 from pathlib import Path
 from typing import Annotated, TextIO
 
@@ -25,6 +26,7 @@
 )
 
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
+from amfv_datasets.scraping.drugs_com import scrape_drugs_com
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
 
@@ -34,6 +36,7 @@ class ScraperSource(StrEnum):
 
     ALL = "all"
     NICE = "nice"
+    DRUGSCOM = "drugscom"
 
 
 class OutputFormat(StrEnum):
@@ -61,20 +64,50 @@ def scrape_documents(
             implemented source.
         documents: Number of documents to scrape. When unset, each source runs
             until it is exhausted (default: None).
-        link_mode: Whether links are kept as markdown links or stripped to their
-            visible text.
+        link_mode: Whether links are kept as markdown links or stripped to
+            their visible text.
         url: Source URL to scrape as a single document (default: None).
     """
     if documents is not None and documents < 1:
         raise ValueError(f"documents must be at least 1; got {documents}")
 
+    scrape_runs: list[ScrapeRun] = []
+
     for selected_source in _expand_source(source):
         match selected_source:
             case ScraperSource.NICE:
-                return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+                scrape_runs.append(
+                    scrape_nice(
+                        documents=documents,
+                        link_mode=link_mode,
+                        url=url,
+                    )
+                )
+
+            case ScraperSource.DRUGSCOM:
+                scrape_runs.append(
+                    scrape_drugs_com(
+                        documents=documents,
+                        link_mode=link_mode,
+                        url=url,
+                    )
+                )
+
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
-    raise AssertionError(f"unsupported source: {source}")
+
+    if not scrape_runs:
+        raise AssertionError(f"unsupported source: {source}")
+
+    if len(scrape_runs) == 1:
+        return scrape_runs[0]
+
+    total = None if any(run.total is None for run in scrape_runs) else sum(run.total or 0 for run in scrape_runs)
+
+    return ScrapeRun(
+        documents=chain.from_iterable(run.documents for run in scrape_runs),
+        total=total,
+    )
 
 
 def write_jsonl(documents: Iterable[ScrapedDocument], output: TextIO) -> int:
@@ -125,7 +158,10 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (
+            ScraperSource.NICE,
+            ScraperSource.DRUGSCOM,
+        )
     return (source,)