diff --git a/README.md b/README.md index 9330d9a..2cc5170 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,9 @@ Both are optional: EDGAR works unattributed under the default, saying so once. And filings.xbrl.org, local packages and TAVI/holon JSON need no identity at all. Then load filings from the chat — a ticker, an EDGAR `cik:accession`, a -`lei:`, a local package, or a holon or TAVI by path or URL — and ask for +`lei:`, a local package, or a holon or TAVI by path or URL; a ticker or `cik:accession` loads the filing's +published holon first when the RoboSystems CDN has one, falling back to +EDGAR — and ask for statements, facts by concept and period, calculations, exhibits and text. No graph and no database sits behind any of it: every answer about a filing is read from that filing. The one outward call is `search_filings`, which asks diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py index ae2cbc7..502547d 100644 --- a/tests/test_deserialize.py +++ b/tests/test_deserialize.py @@ -268,6 +268,9 @@ def _model() -> XbrlModel: from_qname="us-gaap:SegmentDomain", to_qname="us-gaap:NorthAmerica", arcrole=f"{DIM}/domain-member", + # The members continue in another role: a cube rebuilt without + # following this loses them. + target_role="http://example.com/role/SegmentMembers", ), ], ), @@ -723,6 +726,17 @@ def test_tavi_gaps_are_declared(model: XbrlModel) -> None: assert gaps.unmapped_label_types == {} +def test_holon_keeps_the_target_role(model: XbrlModel) -> None: + """``xbrldt:targetRole`` rides on the association: the role the next hop of + a hypercube's wiring continues in, without which a cube declared across + roles reads as one role's arcs.""" + got = from_holon_json(to_holon(model)) + definition = next(n for n in got.networks if n.kind == "definition") + by_target = {a.to_qname: a.target_role for a in definition.arcs} + assert by_target["us-gaap:NorthAmerica"] == "http://example.com/role/SegmentMembers" + assert by_target["us-gaap:SegmentTable"] is None + + def test_holon_gaps_are_declared(model: XbrlModel) -> None: got, gaps = from_holon_report(to_holon(model)) reported = " ".join(gaps.missing) diff --git a/tests/test_lpg.py b/tests/test_lpg.py index e7cebec..f8ee547 100644 --- a/tests/test_lpg.py +++ b/tests/test_lpg.py @@ -573,6 +573,29 @@ def test_structures_and_associations(self, model): {"from": structure["identifier"], "to": tables.nodes["Taxonomy"][0]["identifier"]} ] + def test_calculations_1_1_arcs_are_calculation_associations(self, model): + """The 2023 summation-item arcrole is the calculation linkbase too, and + keeps its weight.""" + model.networks[1].arcs[0].arcrole = "https://xbrl.org/2023/arcrole/summation-item" + tables = to_graph_tables(model) + calculation = next( + a for a in tables.nodes["Association"] if a["association_type"] == "Calculation" + ) + assert calculation["arcrole"] == "https://xbrl.org/2023/arcrole/summation-item" + assert calculation["weight"] == 1.0 + + def test_a_type_without_a_qname_keeps_its_namespace(self, model): + """A concept read back from a serialization may carry a type's namespace + and local name but no QName for a prefix the filing never bound; the + projection writes the type from what it has rather than dropping it.""" + concept = model.concepts["us-gaap:Assets"] + concept.item_type = "monetaryItemType" + concept.item_type_qname = None + concept.item_type_namespace = "http://www.xbrl.org/2003/instance" + tables = to_graph_tables(model) + assets = next(e for e in tables.nodes["Element"] if e["qname"] == "us-gaap:Assets") + assert assets["item_type"] == "http://www.xbrl.org/2003/instance#monetaryItemType" + def test_projection_is_deterministic(self, model): first = to_graph_tables(model) second = to_graph_tables(model) diff --git a/tests/test_parse.py b/tests/test_parse.py index ed8a3b6..19d1f1f 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -12,6 +12,7 @@ from xbrlkit.parse import ids from xbrlkit.parse.to_model import ( + _classify_arcrole, _make_period, _make_unit, _measure_token, @@ -250,3 +251,20 @@ def test_sec_ixt_transforms_register(): register_sec_transforms() assert FunctionIxt.ixtNamespaceFunctions[SEC_IXT_NAMESPACE] is not None assert len(FunctionIxt.ixtNamespaceFunctions[SEC_IXT_NAMESPACE]) == len(registry) + + +def test_calculations_1_1_is_the_calculation_linkbase(): + """Calculations 1.1 declares the same roll-ups under the 2023 arcrole; a + filer that adopted it must not come back with no calculation network.""" + assert ( + _classify_arcrole("http://www.xbrl.org/2003/arcrole/summation-item") + == "calculation" + ) + assert ( + _classify_arcrole("https://xbrl.org/2023/arcrole/summation-item") == "calculation" + ) + assert ( + _classify_arcrole("http://www.xbrl.org/2003/arcrole/parent-child") == "presentation" + ) + assert _classify_arcrole("http://xbrl.org/int/dim/arcrole/all") == "definition" + assert _classify_arcrole("http://www.xbrl.org/2003/arcrole/concept-label") is None diff --git a/tests/test_published.py b/tests/test_published.py new file mode 100644 index 0000000..26b1c3a --- /dev/null +++ b/tests/test_published.py @@ -0,0 +1,337 @@ +"""Tests for loading a filing from its published representations. + +The RoboSystems public data CDN writes every processed SEC filing as a holon +beside the document as filed, under ``{year}/{cik}/{accession}/``, with a +per-filer catalog under ``companies/``. A ticker or ``cik:accession`` loads +that holon when there is one — in a fraction of the time Arelle takes and +with no taxonomy fetch — and falls back to EDGAR when there is not. These +tests stand a local server in for the CDN and stand a sentinel in for EDGAR, +so each path is seen to be taken, or not. +""" + +from __future__ import annotations + +import functools +import http.server +import json +import socketserver +import threading +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.test_deserialize import _model +from xbrlkit.config import Config +from xbrlkit.model import Concept, Label, XbrlFact, XbrlModel +from xbrlkit.serialize import to_holon +from xbrlkit.serve import tools +from xbrlkit.serve.session import FilingSession, PublishedFiling + +US_GAAP = "http://fasb.org/us-gaap/2024" +STANDARD = "http://www.xbrl.org/2003/role/label" +ACCESSION = "0000000000-24-000001" +CIK = "0001234567" +FRAGMENT = ( + "

The Company leases office space under operating leases that expire at " + "various dates through 2031, with renewal options at the Company's discretion " + "and no residual value guarantees on any of them.

" +) + + +def _model_with_text() -> XbrlModel: + """The deserialize fixture plus one text block whose value, as a published + holon carries it, is the URL of its fragment.""" + model = _model() + model.concepts["us-gaap:LesseeOperatingLeasesTextBlock"] = Concept( + qname="us-gaap:LesseeOperatingLeasesTextBlock", + namespace=US_GAAP, + name="LesseeOperatingLeasesTextBlock", + period_type="duration", + is_textblock=True, + is_text_fact=True, + pref_label="Leases", + labels=[Label(value="Leases", role=STANDARD, language="en-US")], + ) + model.facts.append( + XbrlFact( + id="t1", + concept_qname="us-gaap:LesseeOperatingLeasesTextBlock", + period_id=model.periods[1].id, + entity_cik=CIK, + value_str="__FRAGMENT_URL__", + value_kind="text", + ) + ) + return model + + +@pytest.fixture +def cdn(tmp_path: Path) -> Iterator[str]: + """A local stand-in for the public data CDN: the filer's catalog, the + filing's folder with its manifest, holon, document and one fragment.""" + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=str(tmp_path) + ) + with socketserver.TCPServer(("127.0.0.1", 0), handler) as httpd: + base = f"http://127.0.0.1:{httpd.server_address[1]}" + folder = f"{base}/2024/{CIK}/{ACCESSION}" + root = tmp_path / "2024" / CIK / ACCESSION + root.mkdir(parents=True) + (root / "fact_abc.html").write_text(FRAGMENT) + model = _model_with_text() + holon = to_holon(model).replace("__FRAGMENT_URL__", f"{folder}/fact_abc.html") + (root / "holon.jsonld").write_text(holon) + (root / "acme-20241231.htm").write_text( + "

Item 1. Business

Acme makes widgets and sells " + "them everywhere widgets are wanted, which is most places.

" + f"

Note 5. Leases

{FRAGMENT}" + ) + representations = [ + { + "kind": "holon", + "name": "holon.jsonld", + "media_type": "application/ld+json", + "url": f"{folder}/holon.jsonld", + }, + { + "kind": "document", + "name": "acme-20241231.htm", + "media_type": "text/html", + "url": f"{folder}/acme-20241231.htm", + }, + ] + (root / "manifest.json").write_text( + json.dumps({"representations": representations}) + ) + companies = tmp_path / "companies" + companies.mkdir() + (companies / "acme.json").write_text( + json.dumps( + { + "ticker": "ACME", + "cik": CIK, + "filings": [ + { + "accession": ACCESSION, + "form": "10-K", + "filing_date": "2025-02-14", + "folder": folder, + "representations": representations, + }, + { + "accession": "0000000000-23-000001", + "form": "10-K", + "filing_date": "2024-02-14", + "folder": None, + "representations": [], + }, + ], + } + ) + ) + # A filer whose newest 10-K predates the artifacts: no folder, nothing to load. + (companies / "olde.json").write_text( + json.dumps( + { + "ticker": "OLDE", + "filings": [ + { + "accession": "0000000000-23-000009", + "form": "10-K", + "folder": None, + "representations": [], + }, + { + "accession": "0000000000-22-000009", + "form": "10-K", + "folder": folder, + "representations": representations, + }, + ], + } + ) + ) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + yield base + finally: + httpd.shutdown() + + +class _NoEdgar: + """EDGAR must not be touched when the published filing serves.""" + + def __init__(self, *args, **kwargs) -> None: + raise AssertionError("EDGAR was consulted for a published filing") + + +@pytest.fixture +def no_edgar(monkeypatch: pytest.MonkeyPatch) -> None: + import xbrlkit.edgar + + monkeypatch.setattr(xbrlkit.edgar, "EdgarClient", _NoEdgar) + + +def _session(cdn: str, **overrides) -> FilingSession: + return FilingSession(Config(artifacts_base_url=cdn, **overrides)) + + +@pytest.mark.unit +def test_a_ticker_loads_the_published_holon_and_its_document( + cdn: str, no_edgar +) -> None: + session = _session(cdn) + try: + loaded = session.load("ACME") + assert loaded.id == ACCESSION + assert loaded.source_kind == "holon" + assert loaded.has_xbrl is True + # The document as filed came with it: the text tools read the whole + # filing, not only the tagged blocks. + assert loaded.has_document is True + assert loaded.model.filing.document_name == "acme-20241231.htm" + assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0 + hits = tools.search_text(loaded, "widgets") + assert hits["hits"], hits + # The text block's fragment was fetched and inlined. + block = next( + f + for f in loaded.model.facts + if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock" + ) + assert block.value_str == FRAGMENT + finally: + session.close() + + +@pytest.mark.unit +def test_a_form_that_is_not_the_newest_still_resolves_by_form( + cdn: str, no_edgar +) -> None: + """``ACME 10-K`` names the form; the newest filing of that form decides.""" + session = _session(cdn) + try: + assert session.load("ACME 10-K").id == ACCESSION + finally: + session.close() + + +@pytest.mark.unit +def test_cik_and_accession_probe_the_published_folder(cdn: str, no_edgar) -> None: + session = _session(cdn) + try: + loaded = session.load(f"1234567:{ACCESSION}") + assert loaded.id == ACCESSION + assert loaded.source_kind == "holon" + finally: + session.close() + + +@pytest.mark.unit +def test_the_newest_filing_decides_never_an_older_one_with_a_holon( + cdn: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """OLDE's newest 10-K predates the artifacts. The answer is EDGAR for that + filing, not the older filing that happens to be published.""" + from types import SimpleNamespace + + sentinel = SimpleNamespace(id="0000000000-23-000009") + calls: list[tuple[str, str]] = [] + + def fake_edgar(self, cik, accession, source): + calls.append((cik, accession)) + return sentinel + + monkeypatch.setattr(FilingSession, "_load_edgar", fake_edgar) + + class Client: + def __init__(self, config=None) -> None: + pass + + def ticker_to_cik(self, ticker): + return "0000000042" + + def list_filings(self, cik, forms=None): + return [SimpleNamespace(accession="0000000000-23-000009")] + + import xbrlkit.edgar + + monkeypatch.setattr(xbrlkit.edgar, "EdgarClient", Client) + session = _session(cdn) + try: + assert session._published_by_ticker("OLDE", "10-K") is None + assert session.load("OLDE") is sentinel + assert calls == [("0000000042", "0000000000-23-000009")] + finally: + session.close() + + +@pytest.mark.unit +def test_an_unknown_filer_or_accession_falls_back_to_edgar(cdn: str) -> None: + session = _session(cdn) + try: + assert session._published_by_ticker("NOPE", "10-K") is None + assert session._published_by_accession("0000000042", "0000000042-25-000001") is None + assert session._published_by_accession("0000000042", "not-an-accession") is None + finally: + session.close() + + +@pytest.mark.unit +def test_no_base_url_means_no_lookup(cdn: str) -> None: + session = _session("") + try: + assert session._published_by_ticker("ACME", "10-K") is None + assert session._published_by_accession("1234567", ACCESSION) is None + finally: + session.close() + + +@pytest.mark.unit +def test_fragments_stay_urls_when_fetching_is_off(cdn: str, no_edgar) -> None: + session = _session(cdn, fetch_external_text=False) + try: + loaded = session.load("ACME") + block = next( + f + for f in loaded.model.facts + if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock" + ) + assert block.value_str.startswith("http://127.0.0.1:") + finally: + session.close() + + +@pytest.mark.unit +def test_a_missing_fragment_stays_a_url_and_the_load_succeeds( + cdn: str, no_edgar, tmp_path: Path +) -> None: + (tmp_path / "2024" / CIK / ACCESSION / "fact_abc.html").unlink() + session = _session(cdn) + try: + loaded = session.load("ACME") + block = next( + f + for f in loaded.model.facts + if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock" + ) + assert block.value_str.endswith("/fact_abc.html") + finally: + session.close() + + +@pytest.mark.unit +def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None: + from xbrlkit.serve.session import _published_from + + assert ( + _published_from("acc", [{"kind": "document", "url": "http://x/doc.htm"}], None) + is None + ) + published = _published_from( + "acc", [{"kind": "holon", "name": "holon.jsonld"}], "http://x/f/" + ) + assert published == PublishedFiling( + accession="acc", holon_url="http://x/f/holon.jsonld" + ) diff --git a/xbrlkit/config.py b/xbrlkit/config.py index c3c679d..27578fc 100644 --- a/xbrlkit/config.py +++ b/xbrlkit/config.py @@ -83,6 +83,25 @@ class Config: os.environ.get("XBRLKIT_ARELLE_OFFLINE", "").lower() in ("1", "true", "yes") ) ) + # Where a filing's published representations live — the RoboSystems public + # data CDN, which writes every processed SEC filing as a holon, a TAVI model + # and the document as filed, under ``{year}/{cik}/{accession}/``, with a + # per-filer catalog under ``companies/``. A ticker or ``cik:accession`` loads + # the published holon when there is one, in under a second, and falls back + # to EDGAR and Arelle when there is not. Empty disables the lookup. + artifacts_base_url: str = field( + default_factory=lambda: os.environ.get( + "XBRLKIT_ARTIFACTS_URL", "https://public.robosystems.ai" + ).rstrip("/") + ) + # A published holon carries a large text block as the URL of its fragment + # rather than the text; fetching those on load gives the text tools the + # block itself. Off, a fragment reads as its URL. + fetch_external_text: bool = field( + default_factory=lambda: ( + os.environ.get("XBRLKIT_FETCH_TEXT", "true").lower() not in ("0", "false", "no") + ) + ) @property def arelle_cache_dir(self) -> Path: diff --git a/xbrlkit/deserialize/holon.py b/xbrlkit/deserialize/holon.py index 5a198a3..cc5ec9c 100644 --- a/xbrlkit/deserialize/holon.py +++ b/xbrlkit/deserialize/holon.py @@ -97,6 +97,8 @@ PARENT_CHILD_ARCROLE = "http://www.xbrl.org/2003/arcrole/parent-child" SUMMATION_ITEM_ARCROLE = "http://www.xbrl.org/2003/arcrole/summation-item" +# Calculations 1.1: the same roll-ups under the 2023 arcrole. +SUMMATION_ITEM_11_ARCROLE = "https://xbrl.org/2023/arcrole/summation-item" DIMENSION_ARCROLE_BASE = "http://xbrl.org/int/dim/arcrole/" # The arcrole an arc takes when the association did not carry its own. A # definition network has no single arcrole — its arcs are the dimensional @@ -758,7 +760,7 @@ def _network_kind(node: Mapping[str, Any]) -> NetworkKind | None: if declared in ("presentation", "calculation", "definition"): return declared arcrole = _text(node.get("arcrole")) or "" - if arcrole == SUMMATION_ITEM_ARCROLE: + if arcrole in (SUMMATION_ITEM_ARCROLE, SUMMATION_ITEM_11_ARCROLE): return "calculation" if arcrole == PARENT_CHILD_ARCROLE: return "presentation" @@ -874,6 +876,7 @@ def _networks( weight=_float(node.get("weight")), preferred_label=_text(node.get("preferredLabelRole")), is_root=source in roots, + target_role=_text(_prop(node, "targetRole")), ) for source, target, node in edges ], diff --git a/xbrlkit/parse/to_model.py b/xbrlkit/parse/to_model.py index 35decab..e2ad756 100644 --- a/xbrlkit/parse/to_model.py +++ b/xbrlkit/parse/to_model.py @@ -527,10 +527,16 @@ def _value_str(fact: object) -> str | None: def _classify_arcrole(arcrole: str) -> NetworkKind | None: - """Map an arcrole to a linkbase kind, or ``None`` to skip it.""" + """Map an arcrole to a linkbase kind, or ``None`` to skip it. + + Calculations 1.1 (the 2023 ``summation-item`` arcrole) is the calculation + linkbase as much as the 2003 one is: a filer that adopted it declares the + same roll-ups under the newer arcrole, and reading those as definition + arcs left every such filing with no calculation network and no footing. + """ if arcrole == XbrlConst.parentChild: return "presentation" - if arcrole == XbrlConst.summationItem: + if arcrole in (XbrlConst.summationItem, XbrlConst.summationItem11): return "calculation" if arcrole in (XbrlConst.conceptLabel, XbrlConst.conceptReference): return None diff --git a/xbrlkit/serialize/graph.py b/xbrlkit/serialize/graph.py index 5a5a7bd..66ae2ef 100644 --- a/xbrlkit/serialize/graph.py +++ b/xbrlkit/serialize/graph.py @@ -605,6 +605,11 @@ def _add_structures( if arc.arcrole: g.add((a_uri, XLINK.arcrole, _arcrole_uri(arc.arcrole))) g.add((a_uri, XLINK.role, URIRef(st.role_uri))) + # xbrldt:targetRole — the role the next hop of a hypercube's wiring + # continues in. A cube rebuilt without it loses every axis or + # member the filer declared in another role. + if arc.target_role: + g.add((a_uri, RS.targetRole, Literal(arc.target_role))) if arc.order is not None: g.add( ( diff --git a/xbrlkit/serialize/lpg.py b/xbrlkit/serialize/lpg.py index 6447cc4..6f9f42c 100644 --- a/xbrlkit/serialize/lpg.py +++ b/xbrlkit/serialize/lpg.py @@ -61,6 +61,9 @@ CIK_SCHEME = "http://www.sec.gov/CIK" PARENT_CHILD = "http://www.xbrl.org/2003/arcrole/parent-child" SUMMATION_ITEM = "http://www.xbrl.org/2003/arcrole/summation-item" +# Calculations 1.1 declares the same roll-ups under a newer arcrole. +SUMMATION_ITEM_11 = "https://xbrl.org/2023/arcrole/summation-item" +SUMMATION_ITEMS = (SUMMATION_ITEM, SUMMATION_ITEM_11) Row = dict[str, Any] @@ -268,7 +271,12 @@ def _element(self, qname: str) -> str | None: substitution_group=_qname_uri( concept.substitution_group, concept.substitution_group_namespace ), - item_type=_qname_uri(concept.item_type_qname, concept.item_type_namespace), + # The type's namespace and local name from whichever field carries the + # name: a concept read back from a serialization may keep the namespace + # and the local name but no QName for a prefix the filing never bound. + item_type=_qname_uri( + concept.item_type_qname or concept.item_type, concept.item_type_namespace + ), ) self._labels_and_references(concept, element_id, uri) return element_id @@ -355,7 +363,7 @@ def _association( self._associations.add(association_id) if arcrole == PARENT_CHILD: association_type = "Presentation" - elif arcrole == SUMMATION_ITEM: + elif arcrole in SUMMATION_ITEMS: association_type = "Calculation" else: association_type = "Other" @@ -366,7 +374,7 @@ def _association( order_value=order_value, association_type=association_type, weight=float(arc.weight) - if arcrole == SUMMATION_ITEM and arc.weight is not None + if arcrole in SUMMATION_ITEMS and arc.weight is not None else None, root=arc.is_root, preferred_label=arc.preferred_label, diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py index 5967fe8..7c85677 100644 --- a/xbrlkit/serve/session.py +++ b/xbrlkit/serve/session.py @@ -17,6 +17,7 @@ import shutil import tempfile import threading +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import date from pathlib import Path @@ -48,6 +49,10 @@ _PLAIN_SUFFIXES = {".txt", ".md"} # A report serialized as JSON — read into the model here, never by Arelle. _JSON_SUFFIXES = {".json", ".jsonld"} +# The published folder is keyed by filing year, ten-digit CIK and accession; the +# accession's middle segment is the year it was assigned. +_ACCESSION_YEAR_RE = re.compile(r"^\d{10}-(\d{2})-\d{6}$") +_EXTERNAL_TEXT_WORKERS = 8 # What a document-only filing can be read from. A PDF (an ARS, an SEC comment # letter) is a document EDGAR holds and this cannot read; it is named as such # rather than loaded empty. @@ -154,6 +159,41 @@ class NoXbrlFound(SourceError): """ +@dataclass +class PublishedFiling: + """A filing as the public data CDN lists it: the holon to load, and the + document as filed when it was published beside it.""" + + accession: str + holon_url: str + document_url: str | None = None + + +def _published_from( + accession: str | None, representations: Any, folder: str | None +) -> PublishedFiling | None: + """The published filing a catalog entry or manifest describes, or ``None`` + when it lists no holon.""" + holon = document = None + for rep in representations if isinstance(representations, list) else []: + if not isinstance(rep, dict): + continue + url = rep.get("url") or ( + f"{folder.rstrip('/')}/{rep['name']}" if folder and rep.get("name") else None + ) + if not url: + continue + if rep.get("kind") == "holon": + holon = url + elif rep.get("kind") == "document": + document = url + if not holon: + return None + return PublishedFiling( + accession=accession or "", holon_url=holon, document_url=document + ) + + class FilingSession: """The filings loaded into one server process, by id.""" @@ -302,7 +342,9 @@ def _load_local(self, path: Path, source: str) -> LoadedFiling: return self._document_only(target, source, accession, package_dir) return self._finish(_local_id(path, model), source, model, target, package_dir) - def _load_json(self, path: Path, source: str) -> LoadedFiling: + def _load_json( + self, path: Path, source: str, document: Path | None = None + ) -> LoadedFiling: """A JSON file, read into the model without Arelle. Four of the JSON shapes xbrlkit knows are read directly: the parse saved @@ -334,11 +376,17 @@ def _load_json(self, path: Path, source: str) -> LoadedFiling: except (ClawDogError, TaviError, HolonError, ValueError) as exc: raise SourceError(f"{path} could not be read as {kind}: {exc}") from exc target: Path | None = None - if model.filing.primary_document: + if document is not None: + target = document + model.filing.document_name = document.name + elif model.filing.primary_document: candidate = path.parent / model.filing.primary_document if candidate.is_file(): target = candidate model = _enrich_from_dei(model) + inlined = self._inline_external_text(model) + if inlined: + logger.info("inlined %d text block fragments for %s", inlined, source) served = kind in ("tavi", "holon", "clawdog") return self._finish( _local_id(path, model), @@ -371,11 +419,14 @@ def _load_url(self, url: str) -> LoadedFiling: target = self._fetch(url) return self._finish(accession, url, model, target, None) - def _fetch(self, url: str) -> Path: - """The document at ``url``, saved beside this session's other work.""" + def _fetch(self, url: str, into: Path | None = None) -> Path: + """The document at ``url``, saved beside this session's other work — under + ``into`` when two filings would otherwise share a file name.""" resp = requests.get(url, headers=self.config.headers, timeout=60) resp.raise_for_status() - target = self._tmp / Path(url.split("?", 1)[0]).name + folder = into or self._tmp + folder.mkdir(parents=True, exist_ok=True) + target = folder / Path(url.split("?", 1)[0]).name target.write_bytes(resp.content) return target @@ -493,6 +544,10 @@ def _load_edgar(self, cik: str, accession: str, source: str) -> LoadedFiling: from xbrlkit.cli import entity_identity, filing_meta from xbrlkit.edgar import EdgarClient, download_filing, download_primary_document + published = self._published_by_accession(cik, accession) + if published is not None: + return self._load_published(published, source) + client = EdgarClient(config=self.config) ref = client.get_filing_ref(cik, accession) if not ref.is_xbrl: @@ -657,6 +712,10 @@ def _load_filings_org( def _load_ticker(self, ticker: str, form: str, source: str) -> LoadedFiling: from xbrlkit.edgar import EdgarClient + published = self._published_by_ticker(ticker, form) + if published is not None: + return self._load_published(published, source) + client = EdgarClient(config=self.config) cik = client.ticker_to_cik(ticker) refs = client.list_filings(cik, forms=[form.upper()]) @@ -664,6 +723,113 @@ def _load_ticker(self, ticker: str, form: str, source: str) -> LoadedFiling: raise SourceError(f"No {form.upper()} filings on EDGAR for {ticker.upper()}.") return self._load_edgar(cik, refs[0].accession, source) + # -- the published representations ------------------------------------------ + + def _published_by_ticker(self, ticker: str, form: str) -> PublishedFiling | None: + """The filer's newest filing of ``form`` on the public catalog, when that + filing has a published holon. + + The newest filing decides: one that predates the artifacts has no holon, + and the answer is then EDGAR, never an older filing that happens to have + one. + """ + base = self.config.artifacts_base_url + if not base: + return None + catalog = self._get_json(f"{base}/companies/{ticker.lower()}.json") + if not isinstance(catalog, dict): + return None + wanted = form.upper() + for filing in catalog.get("filings") or []: + if not isinstance(filing, dict) or (filing.get("form") or "").upper() != wanted: + continue + return _published_from( + filing.get("accession"), filing.get("representations"), filing.get("folder") + ) + return None + + def _published_by_accession(self, cik: str, accession: str) -> PublishedFiling | None: + """The filing's published folder, probed by its manifest.""" + base = self.config.artifacts_base_url + match = _ACCESSION_YEAR_RE.match(accession) + if not base or not match: + return None + folder = f"{base}/20{match.group(1)}/{cik.zfill(10)}/{accession}" + manifest = self._get_json(f"{folder}/manifest.json") + if not isinstance(manifest, dict): + return None + return _published_from(accession, manifest.get("representations"), folder) + + def _get_json(self, url: str) -> Any: + """A small JSON object from the CDN, or ``None`` for anything but a clean + 200 — a missing object answers 403 there, and either way the fallback is + EDGAR.""" + try: + resp = requests.get(url, headers=self.config.headers, timeout=10) + if resp.status_code != 200: + return None + return resp.json() + except (requests.RequestException, ValueError): + return None + + def _load_published(self, published: PublishedFiling, source: str) -> LoadedFiling: + """The filing from its published holon, with the document as filed beside + it when the CDN has that too — the same shape an EDGAR load gives, in a + fraction of the time and with no Arelle.""" + into = self._tmp / (published.accession or Path(published.holon_url).stem) + holon = self._fetch(published.holon_url, into=into) + document: Path | None = None + if published.document_url: + try: + document = self._fetch(published.document_url, into=into) + except requests.RequestException as exc: + logger.warning("published document unavailable for %s: %s", source, exc) + logger.info("loading %s from its published holon", source) + return self._load_json(holon, source, document=document) + + def _inline_external_text(self, model: XbrlModel) -> int: + """Replace a text block's fragment URL with the fragment. + + The published holon carries a large text block as the URL of the fragment + the platform stored beside it. The text tools want the text, so the + fragments are fetched on load, in parallel; one that cannot be fetched + stays a URL rather than failing the load. + """ + if not self.config.fetch_external_text: + return 0 + pending: list[XbrlFact] = [] + for fact in model.facts: + if fact.value_kind != "text" or not (fact.value_str or "").startswith( + ("http://", "https://") + ): + continue + concept = model.concepts.get(fact.concept_qname) + if concept is not None and concept.is_textblock: + pending.append(fact) + if not pending: + return 0 + + def fetch(url: str) -> str | None: + try: + resp = requests.get( + url, headers=self.config.headers, timeout=self.config.request_timeout + ) + resp.raise_for_status() + return resp.text + except requests.RequestException as exc: + logger.warning("text block fragment unavailable: %s (%s)", url, exc) + return None + + with ThreadPoolExecutor(max_workers=_EXTERNAL_TEXT_WORKERS) as pool: + bodies = list(pool.map(fetch, [fact.value_str or "" for fact in pending])) + inlined = 0 + for fact, body in zip(pending, bodies, strict=True): + if body is not None: + fact.value_str = body + fact.raw_value = body + inlined += 1 + return inlined + def _parse( self, target: Path | str,