Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions docs/registry.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,15 @@ architecture class the census counts.

Exit 0 means every newest model is mapped; **exit 1 is the alert** — a class arrived that
no backend accepts. One kind of unmapped class is reported without the alert: a class whose
models all load through custom modeling code (`auto_map` in the config, or a runtime that is
not Transformers, so Transformers has no implementation of it either) and that fewer than 3
uploaders publish. Research one-offs like that arrive every day (`Ihatetomatoes/nanoBeard-sloop-14M`
models all load through custom modeling code and that fewer than 3 uploaders publish. Custom
code is any of three things, and in each Transformers has no implementation of the class
either: `auto_map` in the config; a runtime library that is not Transformers; or a repo with
no library at all that ships its own top-level `modeling_<name>.py` and is not tagged
`transformers` (`Climb-Ai/Climb-1-Toki`: `MicroLlamaForCausalLM` is a bare `torch.nn.Module`
in `modeling_micro_llama.py`, with `model_type: llama` and no `auto_map`, engine#1922). The
third rule reads the repo's file names, which is why the listing is asked for `full=true`.
The Hub marks a repo it can load with the library as a Transformers model, so a new class
Transformers really has still alerts. Research one-offs like that arrive every day (`Ihatetomatoes/nanoBeard-sloop-14M`
is a raw `pytorch` nanoGPT checkpoint whose config says `architectures: ["GPT"]` and
`model_type: nanobeard-gpt`, engine#296). It alerts once 3 uploaders use it, and a reviewed
class (`registry/significant.json`) always alerts, unless its entry says `"watch": "report"`:
Expand Down
39 changes: 39 additions & 0 deletions tests/census_watch_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,42 @@ def text_of(models, archs=ARCHS, sig=SIG, seen=()):
rc, res = rc_of([model("z/nano", "Qwen3_5MVLAAbsorbedForCausalLM", library="mlx")])
check("a reviewed class alerts even when not Transformers", rc == 1)

# 4d. engine#1922: a repo with no library at all that ships its own modeling_*.py. The record is
# Climb-Ai/Climb-1-Toki's listing row of 2026-11-10 (the fields this watcher reads): class
# MicroLlamaForCausalLM, `model_type: llama`, no auto_map, and modeling_micro_llama.py where
# that class is a bare torch.nn.Module. It alerted as a new class; it is the same one-off.
def own_code(mid, arch, files=("config.json", "model.safetensors", "modeling_micro_llama.py"), **kw):
m = model(mid, arch, **kw)
m["siblings"] = [{"rfilename": f} for f in files]
return m


TOKI_TAGS = ["pytorch", "safetensors", "llama", "toki-pona", "micro-llama", "text-generation",
"conversational", "license:apache-2.0", "region:us"]
rc, text = text_of([own_code("Climb-Ai/Climb-1-Toki", "MicroLlamaForCausalLM", tags=TOKI_TAGS)])
check("own modeling_*.py, no library -> exit 0, reported",
rc == 0 and "REMOTE-CODE MicroLlamaForCausalLM" in text and "UNCOVERED" not in text)
rc, res = rc_of([own_code(f"u{i}/toki", "MicroLlamaForCausalLM") for i in range(cw.REMOTE_CODE_SPREAD)])
check("own-code class from REMOTE_CODE_SPREAD uploaders -> exit 1", rc == 1 and res["remote_code"] == [])
# the rule must not quiet a class Transformers has: the Hub marks such a repo as a Transformers model
rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", tags=["transformers"])])
check("own modeling_*.py with the transformers tag still alerts", rc == 1 and res["remote_code"] == [])
rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", library="transformers")])
check("own modeling_*.py with library transformers still alerts", rc == 1 and res["remote_code"] == [])
rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", library="sentence-transformers")])
check("own modeling_*.py under an unknown library still alerts", rc == 1 and res["remote_code"] == [])
# only a top-level modeling_<name>.py counts: not a training script, not a file in a folder
rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", files=("config.json", "train.py", "model.py"))])
check("other python files are not modeling code", rc == 1 and res["remote_code"] == [])
rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", files=("config.json", "src/modeling_x.py"))])
check("a modeling file in a subfolder is not the repo's model class", rc == 1 and res["remote_code"] == [])
rc, res = rc_of([own_code("o/a", "MixForCausalLM"), model("o/b", "MixForCausalLM")])
check("a class with one model that ships no code alerts", rc == 1 and res["remote_code"] == [])
rc, res = rc_of([own_code("o/a", "Qwen3_5MVLAAbsorbedForCausalLM")])
check("a reviewed class alerts even with its own modeling file", rc == 1)
rc, res = rc_of([own_code("o/a", "Qwen3ForCausalLM")])
check("a mapped class stays covered whatever files it ships", rc == 0 and res[cw.COVERED] == ["o/a"])

# 5. A derivative with no config is expected, not an alert.
rc, res = rc_of([model("e/five", tags=["gguf", "text-generation"])])
check("derivative -> no alert, not unverifiable",
Expand Down Expand Up @@ -230,6 +266,9 @@ def fetch(url):
len(out) == 120 and out[0]["id"] == "a/0" and out[119]["id"] == "b/19")
check("sweep uses the cursor, never p=",
len(urls) == 2 and "p=" not in urls[0] and "cursor=PAGE1" in urls[1])
# engine#1922: the file names arrive only with full=true; without them 4d's rule sees nothing
check("sweep asks the listing for file names (full=true) and the config",
"full=true" in urls[0] and "config=true" in urls[0])

fetch, urls = pages([f"a/{i}" for i in range(100)])
out = cw.sweep_newest(5, seen={f"a/{i}" for i in range(95)}, fetch=fetch)
Expand Down
33 changes: 25 additions & 8 deletions tools/census_watch.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@
(1), since that is the code a human has to act on, and the failure is still printed as
`!! FLM WATCH FAILED` for the log grep.

A class whose models all load through custom modeling code (`auto_map` in the config, or a
runtime library that is not Transformers, so Transformers itself has no implementation of
it either) and that fewer than REMOTE_CODE_SPREAD uploaders publish is reported but does
not alert: research one-offs like that arrive every day and are not something a backend
A class whose models all load through custom modeling code (`auto_map` in the config, a
runtime library that is not Transformers, or a repo with no library at all that ships its own
`modeling_*.py`; in each case Transformers itself has no implementation of the class) and
that fewer than REMOTE_CODE_SPREAD uploaders publish is reported but does not alert: research one-offs like that arrive every day and are not something a backend
could map. Once REMOTE_CODE_SPREAD uploaders use it, or it is a reviewed class
(registry/significant.json), it alerts like any other. The exception is a reviewed class
whose entry says `"watch": "report"`: a research architecture reviewed as not an alias that
Expand Down Expand Up @@ -172,6 +172,13 @@ def request_headers():
"mlc-llm", "tensorflow", "tf-keras", "keras", "jax", "flax", "openvino", "coreml",
))

# A repo's own modeling code, by the name Transformers gives such files (`modeling_<name>.py`),
# at the top level of the repo. engine#1922: Climb-Ai/Climb-1-Toki declares
# `architectures: ["MicroLlamaForCausalLM"]` and `model_type: llama` with no `auto_map` and no
# library, and ships `modeling_micro_llama.py`, where that class is a bare `torch.nn.Module`
# with its own tensor names (`layers.0.attn.q_proj`, `token_embd`). It alerted as a new class.
_OWN_MODELING = re.compile(r"^modeling_[A-Za-z0-9_]+\.py$")

COVERED, UNCOVERED, UNVERIFIABLE, DERIVATIVE = "covered", "uncovered", "unverifiable", "derivative"


Expand Down Expand Up @@ -228,13 +235,21 @@ def is_remote_code(model):

A raw checkpoint through a non-Transformers library carries an architecture class no
backend can map (the library's own code runs it), exactly like an `auto_map` model.
A missing `library_name` is unknown, not custom, so it alerts.
A missing `library_name` is unknown, not custom, so it alerts - unless the repo ships its
own `modeling_*.py` and nothing calls it a Transformers model: then the class is whatever
that file defines, and without `auto_map` Transformers cannot even be told to load it.
The Hub puts `library_name: transformers` and the `transformers` tag on a repo it can
load with the library, so a new class Transformers really has keeps alerting.
"""
cfg = model.get("config") or {}
if cfg.get("auto_map"):
return True
transformers = "transformers" in (model.get("tags") or [])
if model.get("library_name") in NON_TRANSFORMERS_LIBRARIES:
return "transformers" not in (model.get("tags") or [])
return not transformers
if not model.get("library_name") and not transformers:
return any(_OWN_MODELING.match((sib or {}).get("rfilename") or "")
for sib in model.get("siblings") or ())
return False


Expand Down Expand Up @@ -299,7 +314,9 @@ def sweep_newest(limit, seen, tag="text-generation", pages_guard=20, fetch=None)
"""
fetch = fetch or hf_get_page
fresh, batch_ids = [], set()
url = f"{API}?pipeline_tag={tag}&sort=createdAt&direction=-1&limit=100&config=true"
# full=true: the listing then carries each repo's file names (`siblings`), which is how
# is_remote_code() sees a repo's own modeling code; config=true alone leaves them out
url = f"{API}?pipeline_tag={tag}&sort=createdAt&direction=-1&limit=100&config=true&full=true"
page = 0
while url and len(fresh) < limit and page < pages_guard:
batch, link = fetch(url)
Expand Down Expand Up @@ -779,7 +796,7 @@ def report(models, res, previously_seen):
elif cls in quiet:
owners = len({i.split("/")[0] for i in ids})
print(f" . REMOTE-CODE {cls}: {len(ids)} model(s) from {owners} uploader(s), e.g. {ids[0]}")
print(f" -> custom modeling code (auto_map, or a non-Transformers runtime); "
print(f" -> custom modeling code (auto_map, a non-Transformers runtime, or the repo's own modeling_*.py); "
f"alerts once {REMOTE_CODE_SPREAD} uploaders use it")
elif cls in res["significant"]:
print(f" !! UNCOVERED {cls} (reviewed, not an alias — docs/arch-gaps.md): "
Expand Down
Loading