diff --git a/README.md b/README.md
index 9330d9a..2cc5170 100644
--- a/README.md
+++ b/README.md
@@ -195,7 +195,9 @@ Both are optional: EDGAR works unattributed under the default, saying so once.
And filings.xbrl.org, local packages and TAVI/holon JSON need no identity at all.
Then load filings from the chat — a ticker, an EDGAR `cik:accession`, a
-`lei:`, a local package, or a holon or TAVI by path or URL — and ask for
+`lei:`, a local package, or a holon or TAVI by path or URL; a ticker or `cik:accession` loads the filing's
+published holon first when the RoboSystems CDN has one, falling back to
+EDGAR — and ask for
statements, facts by concept and period, calculations, exhibits and text.
No graph and no database sits behind any of it: every answer about a filing is
read from that filing. The one outward call is `search_filings`, which asks
diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py
index ae2cbc7..502547d 100644
--- a/tests/test_deserialize.py
+++ b/tests/test_deserialize.py
@@ -268,6 +268,9 @@ def _model() -> XbrlModel:
from_qname="us-gaap:SegmentDomain",
to_qname="us-gaap:NorthAmerica",
arcrole=f"{DIM}/domain-member",
+ # The members continue in another role: a cube rebuilt without
+ # following this loses them.
+ target_role="http://example.com/role/SegmentMembers",
),
],
),
@@ -723,6 +726,17 @@ def test_tavi_gaps_are_declared(model: XbrlModel) -> None:
assert gaps.unmapped_label_types == {}
+def test_holon_keeps_the_target_role(model: XbrlModel) -> None:
+ """``xbrldt:targetRole`` rides on the association: the role the next hop of
+ a hypercube's wiring continues in, without which a cube declared across
+ roles reads as one role's arcs."""
+ got = from_holon_json(to_holon(model))
+ definition = next(n for n in got.networks if n.kind == "definition")
+ by_target = {a.to_qname: a.target_role for a in definition.arcs}
+ assert by_target["us-gaap:NorthAmerica"] == "http://example.com/role/SegmentMembers"
+ assert by_target["us-gaap:SegmentTable"] is None
+
+
def test_holon_gaps_are_declared(model: XbrlModel) -> None:
got, gaps = from_holon_report(to_holon(model))
reported = " ".join(gaps.missing)
diff --git a/tests/test_lpg.py b/tests/test_lpg.py
index e7cebec..f8ee547 100644
--- a/tests/test_lpg.py
+++ b/tests/test_lpg.py
@@ -573,6 +573,29 @@ def test_structures_and_associations(self, model):
{"from": structure["identifier"], "to": tables.nodes["Taxonomy"][0]["identifier"]}
]
+ def test_calculations_1_1_arcs_are_calculation_associations(self, model):
+ """The 2023 summation-item arcrole is the calculation linkbase too, and
+ keeps its weight."""
+ model.networks[1].arcs[0].arcrole = "https://xbrl.org/2023/arcrole/summation-item"
+ tables = to_graph_tables(model)
+ calculation = next(
+ a for a in tables.nodes["Association"] if a["association_type"] == "Calculation"
+ )
+ assert calculation["arcrole"] == "https://xbrl.org/2023/arcrole/summation-item"
+ assert calculation["weight"] == 1.0
+
+ def test_a_type_without_a_qname_keeps_its_namespace(self, model):
+ """A concept read back from a serialization may carry a type's namespace
+ and local name but no QName for a prefix the filing never bound; the
+ projection writes the type from what it has rather than dropping it."""
+ concept = model.concepts["us-gaap:Assets"]
+ concept.item_type = "monetaryItemType"
+ concept.item_type_qname = None
+ concept.item_type_namespace = "http://www.xbrl.org/2003/instance"
+ tables = to_graph_tables(model)
+ assets = next(e for e in tables.nodes["Element"] if e["qname"] == "us-gaap:Assets")
+ assert assets["item_type"] == "http://www.xbrl.org/2003/instance#monetaryItemType"
+
def test_projection_is_deterministic(self, model):
first = to_graph_tables(model)
second = to_graph_tables(model)
diff --git a/tests/test_parse.py b/tests/test_parse.py
index ed8a3b6..19d1f1f 100644
--- a/tests/test_parse.py
+++ b/tests/test_parse.py
@@ -12,6 +12,7 @@
from xbrlkit.parse import ids
from xbrlkit.parse.to_model import (
+ _classify_arcrole,
_make_period,
_make_unit,
_measure_token,
@@ -250,3 +251,20 @@ def test_sec_ixt_transforms_register():
register_sec_transforms()
assert FunctionIxt.ixtNamespaceFunctions[SEC_IXT_NAMESPACE] is not None
assert len(FunctionIxt.ixtNamespaceFunctions[SEC_IXT_NAMESPACE]) == len(registry)
+
+
+def test_calculations_1_1_is_the_calculation_linkbase():
+ """Calculations 1.1 declares the same roll-ups under the 2023 arcrole; a
+ filer that adopted it must not come back with no calculation network."""
+ assert (
+ _classify_arcrole("http://www.xbrl.org/2003/arcrole/summation-item")
+ == "calculation"
+ )
+ assert (
+ _classify_arcrole("https://xbrl.org/2023/arcrole/summation-item") == "calculation"
+ )
+ assert (
+ _classify_arcrole("http://www.xbrl.org/2003/arcrole/parent-child") == "presentation"
+ )
+ assert _classify_arcrole("http://xbrl.org/int/dim/arcrole/all") == "definition"
+ assert _classify_arcrole("http://www.xbrl.org/2003/arcrole/concept-label") is None
diff --git a/tests/test_published.py b/tests/test_published.py
new file mode 100644
index 0000000..26b1c3a
--- /dev/null
+++ b/tests/test_published.py
@@ -0,0 +1,337 @@
+"""Tests for loading a filing from its published representations.
+
+The RoboSystems public data CDN writes every processed SEC filing as a holon
+beside the document as filed, under ``{year}/{cik}/{accession}/``, with a
+per-filer catalog under ``companies/``. A ticker or ``cik:accession`` loads
+that holon when there is one — in a fraction of the time Arelle takes and
+with no taxonomy fetch — and falls back to EDGAR when there is not. These
+tests stand a local server in for the CDN and stand a sentinel in for EDGAR,
+so each path is seen to be taken, or not.
+"""
+
+from __future__ import annotations
+
+import functools
+import http.server
+import json
+import socketserver
+import threading
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+from tests.test_deserialize import _model
+from xbrlkit.config import Config
+from xbrlkit.model import Concept, Label, XbrlFact, XbrlModel
+from xbrlkit.serialize import to_holon
+from xbrlkit.serve import tools
+from xbrlkit.serve.session import FilingSession, PublishedFiling
+
+US_GAAP = "http://fasb.org/us-gaap/2024"
+STANDARD = "http://www.xbrl.org/2003/role/label"
+ACCESSION = "0000000000-24-000001"
+CIK = "0001234567"
+FRAGMENT = (
+ "
The Company leases office space under operating leases that expire at "
+ "various dates through 2031, with renewal options at the Company's discretion "
+ "and no residual value guarantees on any of them.
"
+)
+
+
+def _model_with_text() -> XbrlModel:
+ """The deserialize fixture plus one text block whose value, as a published
+ holon carries it, is the URL of its fragment."""
+ model = _model()
+ model.concepts["us-gaap:LesseeOperatingLeasesTextBlock"] = Concept(
+ qname="us-gaap:LesseeOperatingLeasesTextBlock",
+ namespace=US_GAAP,
+ name="LesseeOperatingLeasesTextBlock",
+ period_type="duration",
+ is_textblock=True,
+ is_text_fact=True,
+ pref_label="Leases",
+ labels=[Label(value="Leases", role=STANDARD, language="en-US")],
+ )
+ model.facts.append(
+ XbrlFact(
+ id="t1",
+ concept_qname="us-gaap:LesseeOperatingLeasesTextBlock",
+ period_id=model.periods[1].id,
+ entity_cik=CIK,
+ value_str="__FRAGMENT_URL__",
+ value_kind="text",
+ )
+ )
+ return model
+
+
+@pytest.fixture
+def cdn(tmp_path: Path) -> Iterator[str]:
+ """A local stand-in for the public data CDN: the filer's catalog, the
+ filing's folder with its manifest, holon, document and one fragment."""
+ handler = functools.partial(
+ http.server.SimpleHTTPRequestHandler, directory=str(tmp_path)
+ )
+ with socketserver.TCPServer(("127.0.0.1", 0), handler) as httpd:
+ base = f"http://127.0.0.1:{httpd.server_address[1]}"
+ folder = f"{base}/2024/{CIK}/{ACCESSION}"
+ root = tmp_path / "2024" / CIK / ACCESSION
+ root.mkdir(parents=True)
+ (root / "fact_abc.html").write_text(FRAGMENT)
+ model = _model_with_text()
+ holon = to_holon(model).replace("__FRAGMENT_URL__", f"{folder}/fact_abc.html")
+ (root / "holon.jsonld").write_text(holon)
+ (root / "acme-20241231.htm").write_text(
+ "Item 1. Business
Acme makes widgets and sells "
+ "them everywhere widgets are wanted, which is most places.
"
+ f"Note 5. Leases
{FRAGMENT}"
+ )
+ representations = [
+ {
+ "kind": "holon",
+ "name": "holon.jsonld",
+ "media_type": "application/ld+json",
+ "url": f"{folder}/holon.jsonld",
+ },
+ {
+ "kind": "document",
+ "name": "acme-20241231.htm",
+ "media_type": "text/html",
+ "url": f"{folder}/acme-20241231.htm",
+ },
+ ]
+ (root / "manifest.json").write_text(
+ json.dumps({"representations": representations})
+ )
+ companies = tmp_path / "companies"
+ companies.mkdir()
+ (companies / "acme.json").write_text(
+ json.dumps(
+ {
+ "ticker": "ACME",
+ "cik": CIK,
+ "filings": [
+ {
+ "accession": ACCESSION,
+ "form": "10-K",
+ "filing_date": "2025-02-14",
+ "folder": folder,
+ "representations": representations,
+ },
+ {
+ "accession": "0000000000-23-000001",
+ "form": "10-K",
+ "filing_date": "2024-02-14",
+ "folder": None,
+ "representations": [],
+ },
+ ],
+ }
+ )
+ )
+ # A filer whose newest 10-K predates the artifacts: no folder, nothing to load.
+ (companies / "olde.json").write_text(
+ json.dumps(
+ {
+ "ticker": "OLDE",
+ "filings": [
+ {
+ "accession": "0000000000-23-000009",
+ "form": "10-K",
+ "folder": None,
+ "representations": [],
+ },
+ {
+ "accession": "0000000000-22-000009",
+ "form": "10-K",
+ "folder": folder,
+ "representations": representations,
+ },
+ ],
+ }
+ )
+ )
+ threading.Thread(target=httpd.serve_forever, daemon=True).start()
+ try:
+ yield base
+ finally:
+ httpd.shutdown()
+
+
+class _NoEdgar:
+ """EDGAR must not be touched when the published filing serves."""
+
+ def __init__(self, *args, **kwargs) -> None:
+ raise AssertionError("EDGAR was consulted for a published filing")
+
+
+@pytest.fixture
+def no_edgar(monkeypatch: pytest.MonkeyPatch) -> None:
+ import xbrlkit.edgar
+
+ monkeypatch.setattr(xbrlkit.edgar, "EdgarClient", _NoEdgar)
+
+
+def _session(cdn: str, **overrides) -> FilingSession:
+ return FilingSession(Config(artifacts_base_url=cdn, **overrides))
+
+
+@pytest.mark.unit
+def test_a_ticker_loads_the_published_holon_and_its_document(
+ cdn: str, no_edgar
+) -> None:
+ session = _session(cdn)
+ try:
+ loaded = session.load("ACME")
+ assert loaded.id == ACCESSION
+ assert loaded.source_kind == "holon"
+ assert loaded.has_xbrl is True
+ # The document as filed came with it: the text tools read the whole
+ # filing, not only the tagged blocks.
+ assert loaded.has_document is True
+ assert loaded.model.filing.document_name == "acme-20241231.htm"
+ assert tools.fact_grid(loaded, ["us-gaap:Assets"])["rows"][0]["value"] == 1000.0
+ hits = tools.search_text(loaded, "widgets")
+ assert hits["hits"], hits
+ # The text block's fragment was fetched and inlined.
+ block = next(
+ f
+ for f in loaded.model.facts
+ if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock"
+ )
+ assert block.value_str == FRAGMENT
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_a_form_that_is_not_the_newest_still_resolves_by_form(
+ cdn: str, no_edgar
+) -> None:
+ """``ACME 10-K`` names the form; the newest filing of that form decides."""
+ session = _session(cdn)
+ try:
+ assert session.load("ACME 10-K").id == ACCESSION
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_cik_and_accession_probe_the_published_folder(cdn: str, no_edgar) -> None:
+ session = _session(cdn)
+ try:
+ loaded = session.load(f"1234567:{ACCESSION}")
+ assert loaded.id == ACCESSION
+ assert loaded.source_kind == "holon"
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_the_newest_filing_decides_never_an_older_one_with_a_holon(
+ cdn: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """OLDE's newest 10-K predates the artifacts. The answer is EDGAR for that
+ filing, not the older filing that happens to be published."""
+ from types import SimpleNamespace
+
+ sentinel = SimpleNamespace(id="0000000000-23-000009")
+ calls: list[tuple[str, str]] = []
+
+ def fake_edgar(self, cik, accession, source):
+ calls.append((cik, accession))
+ return sentinel
+
+ monkeypatch.setattr(FilingSession, "_load_edgar", fake_edgar)
+
+ class Client:
+ def __init__(self, config=None) -> None:
+ pass
+
+ def ticker_to_cik(self, ticker):
+ return "0000000042"
+
+ def list_filings(self, cik, forms=None):
+ return [SimpleNamespace(accession="0000000000-23-000009")]
+
+ import xbrlkit.edgar
+
+ monkeypatch.setattr(xbrlkit.edgar, "EdgarClient", Client)
+ session = _session(cdn)
+ try:
+ assert session._published_by_ticker("OLDE", "10-K") is None
+ assert session.load("OLDE") is sentinel
+ assert calls == [("0000000042", "0000000000-23-000009")]
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_an_unknown_filer_or_accession_falls_back_to_edgar(cdn: str) -> None:
+ session = _session(cdn)
+ try:
+ assert session._published_by_ticker("NOPE", "10-K") is None
+ assert session._published_by_accession("0000000042", "0000000042-25-000001") is None
+ assert session._published_by_accession("0000000042", "not-an-accession") is None
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_no_base_url_means_no_lookup(cdn: str) -> None:
+ session = _session("")
+ try:
+ assert session._published_by_ticker("ACME", "10-K") is None
+ assert session._published_by_accession("1234567", ACCESSION) is None
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_fragments_stay_urls_when_fetching_is_off(cdn: str, no_edgar) -> None:
+ session = _session(cdn, fetch_external_text=False)
+ try:
+ loaded = session.load("ACME")
+ block = next(
+ f
+ for f in loaded.model.facts
+ if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock"
+ )
+ assert block.value_str.startswith("http://127.0.0.1:")
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_a_missing_fragment_stays_a_url_and_the_load_succeeds(
+ cdn: str, no_edgar, tmp_path: Path
+) -> None:
+ (tmp_path / "2024" / CIK / ACCESSION / "fact_abc.html").unlink()
+ session = _session(cdn)
+ try:
+ loaded = session.load("ACME")
+ block = next(
+ f
+ for f in loaded.model.facts
+ if f.concept_qname == "us-gaap:LesseeOperatingLeasesTextBlock"
+ )
+ assert block.value_str.endswith("/fact_abc.html")
+ finally:
+ session.close()
+
+
+@pytest.mark.unit
+def test_a_manifest_without_a_holon_is_not_a_published_filing() -> None:
+ from xbrlkit.serve.session import _published_from
+
+ assert (
+ _published_from("acc", [{"kind": "document", "url": "http://x/doc.htm"}], None)
+ is None
+ )
+ published = _published_from(
+ "acc", [{"kind": "holon", "name": "holon.jsonld"}], "http://x/f/"
+ )
+ assert published == PublishedFiling(
+ accession="acc", holon_url="http://x/f/holon.jsonld"
+ )
diff --git a/xbrlkit/config.py b/xbrlkit/config.py
index c3c679d..27578fc 100644
--- a/xbrlkit/config.py
+++ b/xbrlkit/config.py
@@ -83,6 +83,25 @@ class Config:
os.environ.get("XBRLKIT_ARELLE_OFFLINE", "").lower() in ("1", "true", "yes")
)
)
+ # Where a filing's published representations live — the RoboSystems public
+ # data CDN, which writes every processed SEC filing as a holon, a TAVI model
+ # and the document as filed, under ``{year}/{cik}/{accession}/``, with a
+ # per-filer catalog under ``companies/``. A ticker or ``cik:accession`` loads
+ # the published holon when there is one, in under a second, and falls back
+ # to EDGAR and Arelle when there is not. Empty disables the lookup.
+ artifacts_base_url: str = field(
+ default_factory=lambda: os.environ.get(
+ "XBRLKIT_ARTIFACTS_URL", "https://public.robosystems.ai"
+ ).rstrip("/")
+ )
+ # A published holon carries a large text block as the URL of its fragment
+ # rather than the text; fetching those on load gives the text tools the
+ # block itself. Off, a fragment reads as its URL.
+ fetch_external_text: bool = field(
+ default_factory=lambda: (
+ os.environ.get("XBRLKIT_FETCH_TEXT", "true").lower() not in ("0", "false", "no")
+ )
+ )
@property
def arelle_cache_dir(self) -> Path:
diff --git a/xbrlkit/deserialize/holon.py b/xbrlkit/deserialize/holon.py
index 5a198a3..cc5ec9c 100644
--- a/xbrlkit/deserialize/holon.py
+++ b/xbrlkit/deserialize/holon.py
@@ -97,6 +97,8 @@
PARENT_CHILD_ARCROLE = "http://www.xbrl.org/2003/arcrole/parent-child"
SUMMATION_ITEM_ARCROLE = "http://www.xbrl.org/2003/arcrole/summation-item"
+# Calculations 1.1: the same roll-ups under the 2023 arcrole.
+SUMMATION_ITEM_11_ARCROLE = "https://xbrl.org/2023/arcrole/summation-item"
DIMENSION_ARCROLE_BASE = "http://xbrl.org/int/dim/arcrole/"
# The arcrole an arc takes when the association did not carry its own. A
# definition network has no single arcrole — its arcs are the dimensional
@@ -758,7 +760,7 @@ def _network_kind(node: Mapping[str, Any]) -> NetworkKind | None:
if declared in ("presentation", "calculation", "definition"):
return declared
arcrole = _text(node.get("arcrole")) or ""
- if arcrole == SUMMATION_ITEM_ARCROLE:
+ if arcrole in (SUMMATION_ITEM_ARCROLE, SUMMATION_ITEM_11_ARCROLE):
return "calculation"
if arcrole == PARENT_CHILD_ARCROLE:
return "presentation"
@@ -874,6 +876,7 @@ def _networks(
weight=_float(node.get("weight")),
preferred_label=_text(node.get("preferredLabelRole")),
is_root=source in roots,
+ target_role=_text(_prop(node, "targetRole")),
)
for source, target, node in edges
],
diff --git a/xbrlkit/parse/to_model.py b/xbrlkit/parse/to_model.py
index 35decab..e2ad756 100644
--- a/xbrlkit/parse/to_model.py
+++ b/xbrlkit/parse/to_model.py
@@ -527,10 +527,16 @@ def _value_str(fact: object) -> str | None:
def _classify_arcrole(arcrole: str) -> NetworkKind | None:
- """Map an arcrole to a linkbase kind, or ``None`` to skip it."""
+ """Map an arcrole to a linkbase kind, or ``None`` to skip it.
+
+ Calculations 1.1 (the 2023 ``summation-item`` arcrole) is the calculation
+ linkbase as much as the 2003 one is: a filer that adopted it declares the
+ same roll-ups under the newer arcrole, and reading those as definition
+ arcs left every such filing with no calculation network and no footing.
+ """
if arcrole == XbrlConst.parentChild:
return "presentation"
- if arcrole == XbrlConst.summationItem:
+ if arcrole in (XbrlConst.summationItem, XbrlConst.summationItem11):
return "calculation"
if arcrole in (XbrlConst.conceptLabel, XbrlConst.conceptReference):
return None
diff --git a/xbrlkit/serialize/graph.py b/xbrlkit/serialize/graph.py
index 5a5a7bd..66ae2ef 100644
--- a/xbrlkit/serialize/graph.py
+++ b/xbrlkit/serialize/graph.py
@@ -605,6 +605,11 @@ def _add_structures(
if arc.arcrole:
g.add((a_uri, XLINK.arcrole, _arcrole_uri(arc.arcrole)))
g.add((a_uri, XLINK.role, URIRef(st.role_uri)))
+ # xbrldt:targetRole — the role the next hop of a hypercube's wiring
+ # continues in. A cube rebuilt without it loses every axis or
+ # member the filer declared in another role.
+ if arc.target_role:
+ g.add((a_uri, RS.targetRole, Literal(arc.target_role)))
if arc.order is not None:
g.add(
(
diff --git a/xbrlkit/serialize/lpg.py b/xbrlkit/serialize/lpg.py
index 6447cc4..6f9f42c 100644
--- a/xbrlkit/serialize/lpg.py
+++ b/xbrlkit/serialize/lpg.py
@@ -61,6 +61,9 @@
CIK_SCHEME = "http://www.sec.gov/CIK"
PARENT_CHILD = "http://www.xbrl.org/2003/arcrole/parent-child"
SUMMATION_ITEM = "http://www.xbrl.org/2003/arcrole/summation-item"
+# Calculations 1.1 declares the same roll-ups under a newer arcrole.
+SUMMATION_ITEM_11 = "https://xbrl.org/2023/arcrole/summation-item"
+SUMMATION_ITEMS = (SUMMATION_ITEM, SUMMATION_ITEM_11)
Row = dict[str, Any]
@@ -268,7 +271,12 @@ def _element(self, qname: str) -> str | None:
substitution_group=_qname_uri(
concept.substitution_group, concept.substitution_group_namespace
),
- item_type=_qname_uri(concept.item_type_qname, concept.item_type_namespace),
+ # The type's namespace and local name from whichever field carries the
+ # name: a concept read back from a serialization may keep the namespace
+ # and the local name but no QName for a prefix the filing never bound.
+ item_type=_qname_uri(
+ concept.item_type_qname or concept.item_type, concept.item_type_namespace
+ ),
)
self._labels_and_references(concept, element_id, uri)
return element_id
@@ -355,7 +363,7 @@ def _association(
self._associations.add(association_id)
if arcrole == PARENT_CHILD:
association_type = "Presentation"
- elif arcrole == SUMMATION_ITEM:
+ elif arcrole in SUMMATION_ITEMS:
association_type = "Calculation"
else:
association_type = "Other"
@@ -366,7 +374,7 @@ def _association(
order_value=order_value,
association_type=association_type,
weight=float(arc.weight)
- if arcrole == SUMMATION_ITEM and arc.weight is not None
+ if arcrole in SUMMATION_ITEMS and arc.weight is not None
else None,
root=arc.is_root,
preferred_label=arc.preferred_label,
diff --git a/xbrlkit/serve/session.py b/xbrlkit/serve/session.py
index 5967fe8..7c85677 100644
--- a/xbrlkit/serve/session.py
+++ b/xbrlkit/serve/session.py
@@ -17,6 +17,7 @@
import shutil
import tempfile
import threading
+from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
@@ -48,6 +49,10 @@
_PLAIN_SUFFIXES = {".txt", ".md"}
# A report serialized as JSON — read into the model here, never by Arelle.
_JSON_SUFFIXES = {".json", ".jsonld"}
+# The published folder is keyed by filing year, ten-digit CIK and accession; the
+# accession's middle segment is the year it was assigned.
+_ACCESSION_YEAR_RE = re.compile(r"^\d{10}-(\d{2})-\d{6}$")
+_EXTERNAL_TEXT_WORKERS = 8
# What a document-only filing can be read from. A PDF (an ARS, an SEC comment
# letter) is a document EDGAR holds and this cannot read; it is named as such
# rather than loaded empty.
@@ -154,6 +159,41 @@ class NoXbrlFound(SourceError):
"""
+@dataclass
+class PublishedFiling:
+ """A filing as the public data CDN lists it: the holon to load, and the
+ document as filed when it was published beside it."""
+
+ accession: str
+ holon_url: str
+ document_url: str | None = None
+
+
+def _published_from(
+ accession: str | None, representations: Any, folder: str | None
+) -> PublishedFiling | None:
+ """The published filing a catalog entry or manifest describes, or ``None``
+ when it lists no holon."""
+ holon = document = None
+ for rep in representations if isinstance(representations, list) else []:
+ if not isinstance(rep, dict):
+ continue
+ url = rep.get("url") or (
+ f"{folder.rstrip('/')}/{rep['name']}" if folder and rep.get("name") else None
+ )
+ if not url:
+ continue
+ if rep.get("kind") == "holon":
+ holon = url
+ elif rep.get("kind") == "document":
+ document = url
+ if not holon:
+ return None
+ return PublishedFiling(
+ accession=accession or "", holon_url=holon, document_url=document
+ )
+
+
class FilingSession:
"""The filings loaded into one server process, by id."""
@@ -302,7 +342,9 @@ def _load_local(self, path: Path, source: str) -> LoadedFiling:
return self._document_only(target, source, accession, package_dir)
return self._finish(_local_id(path, model), source, model, target, package_dir)
- def _load_json(self, path: Path, source: str) -> LoadedFiling:
+ def _load_json(
+ self, path: Path, source: str, document: Path | None = None
+ ) -> LoadedFiling:
"""A JSON file, read into the model without Arelle.
Four of the JSON shapes xbrlkit knows are read directly: the parse saved
@@ -334,11 +376,17 @@ def _load_json(self, path: Path, source: str) -> LoadedFiling:
except (ClawDogError, TaviError, HolonError, ValueError) as exc:
raise SourceError(f"{path} could not be read as {kind}: {exc}") from exc
target: Path | None = None
- if model.filing.primary_document:
+ if document is not None:
+ target = document
+ model.filing.document_name = document.name
+ elif model.filing.primary_document:
candidate = path.parent / model.filing.primary_document
if candidate.is_file():
target = candidate
model = _enrich_from_dei(model)
+ inlined = self._inline_external_text(model)
+ if inlined:
+ logger.info("inlined %d text block fragments for %s", inlined, source)
served = kind in ("tavi", "holon", "clawdog")
return self._finish(
_local_id(path, model),
@@ -371,11 +419,14 @@ def _load_url(self, url: str) -> LoadedFiling:
target = self._fetch(url)
return self._finish(accession, url, model, target, None)
- def _fetch(self, url: str) -> Path:
- """The document at ``url``, saved beside this session's other work."""
+ def _fetch(self, url: str, into: Path | None = None) -> Path:
+ """The document at ``url``, saved beside this session's other work — under
+ ``into`` when two filings would otherwise share a file name."""
resp = requests.get(url, headers=self.config.headers, timeout=60)
resp.raise_for_status()
- target = self._tmp / Path(url.split("?", 1)[0]).name
+ folder = into or self._tmp
+ folder.mkdir(parents=True, exist_ok=True)
+ target = folder / Path(url.split("?", 1)[0]).name
target.write_bytes(resp.content)
return target
@@ -493,6 +544,10 @@ def _load_edgar(self, cik: str, accession: str, source: str) -> LoadedFiling:
from xbrlkit.cli import entity_identity, filing_meta
from xbrlkit.edgar import EdgarClient, download_filing, download_primary_document
+ published = self._published_by_accession(cik, accession)
+ if published is not None:
+ return self._load_published(published, source)
+
client = EdgarClient(config=self.config)
ref = client.get_filing_ref(cik, accession)
if not ref.is_xbrl:
@@ -657,6 +712,10 @@ def _load_filings_org(
def _load_ticker(self, ticker: str, form: str, source: str) -> LoadedFiling:
from xbrlkit.edgar import EdgarClient
+ published = self._published_by_ticker(ticker, form)
+ if published is not None:
+ return self._load_published(published, source)
+
client = EdgarClient(config=self.config)
cik = client.ticker_to_cik(ticker)
refs = client.list_filings(cik, forms=[form.upper()])
@@ -664,6 +723,113 @@ def _load_ticker(self, ticker: str, form: str, source: str) -> LoadedFiling:
raise SourceError(f"No {form.upper()} filings on EDGAR for {ticker.upper()}.")
return self._load_edgar(cik, refs[0].accession, source)
+ # -- the published representations ------------------------------------------
+
+ def _published_by_ticker(self, ticker: str, form: str) -> PublishedFiling | None:
+ """The filer's newest filing of ``form`` on the public catalog, when that
+ filing has a published holon.
+
+ The newest filing decides: one that predates the artifacts has no holon,
+ and the answer is then EDGAR, never an older filing that happens to have
+ one.
+ """
+ base = self.config.artifacts_base_url
+ if not base:
+ return None
+ catalog = self._get_json(f"{base}/companies/{ticker.lower()}.json")
+ if not isinstance(catalog, dict):
+ return None
+ wanted = form.upper()
+ for filing in catalog.get("filings") or []:
+ if not isinstance(filing, dict) or (filing.get("form") or "").upper() != wanted:
+ continue
+ return _published_from(
+ filing.get("accession"), filing.get("representations"), filing.get("folder")
+ )
+ return None
+
+ def _published_by_accession(self, cik: str, accession: str) -> PublishedFiling | None:
+ """The filing's published folder, probed by its manifest."""
+ base = self.config.artifacts_base_url
+ match = _ACCESSION_YEAR_RE.match(accession)
+ if not base or not match:
+ return None
+ folder = f"{base}/20{match.group(1)}/{cik.zfill(10)}/{accession}"
+ manifest = self._get_json(f"{folder}/manifest.json")
+ if not isinstance(manifest, dict):
+ return None
+ return _published_from(accession, manifest.get("representations"), folder)
+
+ def _get_json(self, url: str) -> Any:
+ """A small JSON object from the CDN, or ``None`` for anything but a clean
+ 200 — a missing object answers 403 there, and either way the fallback is
+ EDGAR."""
+ try:
+ resp = requests.get(url, headers=self.config.headers, timeout=10)
+ if resp.status_code != 200:
+ return None
+ return resp.json()
+ except (requests.RequestException, ValueError):
+ return None
+
+ def _load_published(self, published: PublishedFiling, source: str) -> LoadedFiling:
+ """The filing from its published holon, with the document as filed beside
+ it when the CDN has that too — the same shape an EDGAR load gives, in a
+ fraction of the time and with no Arelle."""
+ into = self._tmp / (published.accession or Path(published.holon_url).stem)
+ holon = self._fetch(published.holon_url, into=into)
+ document: Path | None = None
+ if published.document_url:
+ try:
+ document = self._fetch(published.document_url, into=into)
+ except requests.RequestException as exc:
+ logger.warning("published document unavailable for %s: %s", source, exc)
+ logger.info("loading %s from its published holon", source)
+ return self._load_json(holon, source, document=document)
+
+ def _inline_external_text(self, model: XbrlModel) -> int:
+ """Replace a text block's fragment URL with the fragment.
+
+ The published holon carries a large text block as the URL of the fragment
+ the platform stored beside it. The text tools want the text, so the
+ fragments are fetched on load, in parallel; one that cannot be fetched
+ stays a URL rather than failing the load.
+ """
+ if not self.config.fetch_external_text:
+ return 0
+ pending: list[XbrlFact] = []
+ for fact in model.facts:
+ if fact.value_kind != "text" or not (fact.value_str or "").startswith(
+ ("http://", "https://")
+ ):
+ continue
+ concept = model.concepts.get(fact.concept_qname)
+ if concept is not None and concept.is_textblock:
+ pending.append(fact)
+ if not pending:
+ return 0
+
+ def fetch(url: str) -> str | None:
+ try:
+ resp = requests.get(
+ url, headers=self.config.headers, timeout=self.config.request_timeout
+ )
+ resp.raise_for_status()
+ return resp.text
+ except requests.RequestException as exc:
+ logger.warning("text block fragment unavailable: %s (%s)", url, exc)
+ return None
+
+ with ThreadPoolExecutor(max_workers=_EXTERNAL_TEXT_WORKERS) as pool:
+ bodies = list(pool.map(fetch, [fact.value_str or "" for fact in pending]))
+ inlined = 0
+ for fact, body in zip(pending, bodies, strict=True):
+ if body is not None:
+ fact.value_str = body
+ fact.raw_value = body
+ inlined += 1
+ return inlined
+
def _parse(
self,
target: Path | str,