From 2197f107a65f28a3b38a902f6b3ac228970789e8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 10:54:35 -0400 Subject: [PATCH] census-watch: a repo's own modeling_*.py is custom code, so MicroLlamaForCausalLM reports instead of alerting (Fixes #1922) Climb-Ai/Climb-1-Toki declares architectures ["MicroLlamaForCausalLM"] with model_type llama, no auto_map and no library, and ships modeling_micro_llama.py where that class is a bare torch.nn.Module. The custom-code rule knew auto_map and non-Transformers libraries only, so the class alerted as a new one. - is_remote_code(): a repo with no library_name and no transformers tag that ships a top-level modeling_.py is custom code - sweep_newest(): ask the listing for full=true, which is what carries the file names (config=true alone does not) - tests/census_watch_test.py: the recorded row, the spread threshold, and the cases that must keep alerting - docs/registry.md Co-Authored-By: Claude Fable 5.2 --- docs/registry.md | 12 +++++++++--- tests/census_watch_test.py | 39 ++++++++++++++++++++++++++++++++++++++ tools/census_watch.py | 33 ++++++++++++++++++++++++-------- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/docs/registry.md b/docs/registry.md index 72eb2ae4f..fe1be7909 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -128,9 +128,15 @@ architecture class the census counts. Exit 0 means every newest model is mapped; **exit 1 is the alert** — a class arrived that no backend accepts. One kind of unmapped class is reported without the alert: a class whose -models all load through custom modeling code (`auto_map` in the config, or a runtime that is -not Transformers, so Transformers has no implementation of it either) and that fewer than 3 -uploaders publish. Research one-offs like that arrive every day (`Ihatetomatoes/nanoBeard-sloop-14M` +models all load through custom modeling code and that fewer than 3 uploaders publish. Custom +code is any of three things, and in each Transformers has no implementation of the class +either: `auto_map` in the config; a runtime library that is not Transformers; or a repo with +no library at all that ships its own top-level `modeling_.py` and is not tagged +`transformers` (`Climb-Ai/Climb-1-Toki`: `MicroLlamaForCausalLM` is a bare `torch.nn.Module` +in `modeling_micro_llama.py`, with `model_type: llama` and no `auto_map`, engine#1922). The +third rule reads the repo's file names, which is why the listing is asked for `full=true`. +The Hub marks a repo it can load with the library as a Transformers model, so a new class +Transformers really has still alerts. Research one-offs like that arrive every day (`Ihatetomatoes/nanoBeard-sloop-14M` is a raw `pytorch` nanoGPT checkpoint whose config says `architectures: ["GPT"]` and `model_type: nanobeard-gpt`, engine#296). It alerts once 3 uploaders use it, and a reviewed class (`registry/significant.json`) always alerts, unless its entry says `"watch": "report"`: diff --git a/tests/census_watch_test.py b/tests/census_watch_test.py index 9ccd0dbf6..e70f10803 100644 --- a/tests/census_watch_test.py +++ b/tests/census_watch_test.py @@ -141,6 +141,42 @@ def text_of(models, archs=ARCHS, sig=SIG, seen=()): rc, res = rc_of([model("z/nano", "Qwen3_5MVLAAbsorbedForCausalLM", library="mlx")]) check("a reviewed class alerts even when not Transformers", rc == 1) +# 4d. engine#1922: a repo with no library at all that ships its own modeling_*.py. The record is +# Climb-Ai/Climb-1-Toki's listing row of 2026-11-10 (the fields this watcher reads): class +# MicroLlamaForCausalLM, `model_type: llama`, no auto_map, and modeling_micro_llama.py where +# that class is a bare torch.nn.Module. It alerted as a new class; it is the same one-off. +def own_code(mid, arch, files=("config.json", "model.safetensors", "modeling_micro_llama.py"), **kw): + m = model(mid, arch, **kw) + m["siblings"] = [{"rfilename": f} for f in files] + return m + + +TOKI_TAGS = ["pytorch", "safetensors", "llama", "toki-pona", "micro-llama", "text-generation", + "conversational", "license:apache-2.0", "region:us"] +rc, text = text_of([own_code("Climb-Ai/Climb-1-Toki", "MicroLlamaForCausalLM", tags=TOKI_TAGS)]) +check("own modeling_*.py, no library -> exit 0, reported", + rc == 0 and "REMOTE-CODE MicroLlamaForCausalLM" in text and "UNCOVERED" not in text) +rc, res = rc_of([own_code(f"u{i}/toki", "MicroLlamaForCausalLM") for i in range(cw.REMOTE_CODE_SPREAD)]) +check("own-code class from REMOTE_CODE_SPREAD uploaders -> exit 1", rc == 1 and res["remote_code"] == []) +# the rule must not quiet a class Transformers has: the Hub marks such a repo as a Transformers model +rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", tags=["transformers"])]) +check("own modeling_*.py with the transformers tag still alerts", rc == 1 and res["remote_code"] == []) +rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", library="transformers")]) +check("own modeling_*.py with library transformers still alerts", rc == 1 and res["remote_code"] == []) +rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", library="sentence-transformers")]) +check("own modeling_*.py under an unknown library still alerts", rc == 1 and res["remote_code"] == []) +# only a top-level modeling_.py counts: not a training script, not a file in a folder +rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", files=("config.json", "train.py", "model.py"))]) +check("other python files are not modeling code", rc == 1 and res["remote_code"] == []) +rc, res = rc_of([own_code("o/new", "NimbusMindForCausalLM", files=("config.json", "src/modeling_x.py"))]) +check("a modeling file in a subfolder is not the repo's model class", rc == 1 and res["remote_code"] == []) +rc, res = rc_of([own_code("o/a", "MixForCausalLM"), model("o/b", "MixForCausalLM")]) +check("a class with one model that ships no code alerts", rc == 1 and res["remote_code"] == []) +rc, res = rc_of([own_code("o/a", "Qwen3_5MVLAAbsorbedForCausalLM")]) +check("a reviewed class alerts even with its own modeling file", rc == 1) +rc, res = rc_of([own_code("o/a", "Qwen3ForCausalLM")]) +check("a mapped class stays covered whatever files it ships", rc == 0 and res[cw.COVERED] == ["o/a"]) + # 5. A derivative with no config is expected, not an alert. rc, res = rc_of([model("e/five", tags=["gguf", "text-generation"])]) check("derivative -> no alert, not unverifiable", @@ -230,6 +266,9 @@ def fetch(url): len(out) == 120 and out[0]["id"] == "a/0" and out[119]["id"] == "b/19") check("sweep uses the cursor, never p=", len(urls) == 2 and "p=" not in urls[0] and "cursor=PAGE1" in urls[1]) +# engine#1922: the file names arrive only with full=true; without them 4d's rule sees nothing +check("sweep asks the listing for file names (full=true) and the config", + "full=true" in urls[0] and "config=true" in urls[0]) fetch, urls = pages([f"a/{i}" for i in range(100)]) out = cw.sweep_newest(5, seen={f"a/{i}" for i in range(95)}, fetch=fetch) diff --git a/tools/census_watch.py b/tools/census_watch.py index 8869dd883..f2672fc51 100644 --- a/tools/census_watch.py +++ b/tools/census_watch.py @@ -51,10 +51,10 @@ (1), since that is the code a human has to act on, and the failure is still printed as `!! FLM WATCH FAILED` for the log grep. -A class whose models all load through custom modeling code (`auto_map` in the config, or a -runtime library that is not Transformers, so Transformers itself has no implementation of -it either) and that fewer than REMOTE_CODE_SPREAD uploaders publish is reported but does -not alert: research one-offs like that arrive every day and are not something a backend +A class whose models all load through custom modeling code (`auto_map` in the config, a +runtime library that is not Transformers, or a repo with no library at all that ships its own +`modeling_*.py`; in each case Transformers itself has no implementation of the class) and +that fewer than REMOTE_CODE_SPREAD uploaders publish is reported but does not alert: research one-offs like that arrive every day and are not something a backend could map. Once REMOTE_CODE_SPREAD uploaders use it, or it is a reviewed class (registry/significant.json), it alerts like any other. The exception is a reviewed class whose entry says `"watch": "report"`: a research architecture reviewed as not an alias that @@ -172,6 +172,13 @@ def request_headers(): "mlc-llm", "tensorflow", "tf-keras", "keras", "jax", "flax", "openvino", "coreml", )) +# A repo's own modeling code, by the name Transformers gives such files (`modeling_.py`), +# at the top level of the repo. engine#1922: Climb-Ai/Climb-1-Toki declares +# `architectures: ["MicroLlamaForCausalLM"]` and `model_type: llama` with no `auto_map` and no +# library, and ships `modeling_micro_llama.py`, where that class is a bare `torch.nn.Module` +# with its own tensor names (`layers.0.attn.q_proj`, `token_embd`). It alerted as a new class. +_OWN_MODELING = re.compile(r"^modeling_[A-Za-z0-9_]+\.py$") + COVERED, UNCOVERED, UNVERIFIABLE, DERIVATIVE = "covered", "uncovered", "unverifiable", "derivative" @@ -228,13 +235,21 @@ def is_remote_code(model): A raw checkpoint through a non-Transformers library carries an architecture class no backend can map (the library's own code runs it), exactly like an `auto_map` model. - A missing `library_name` is unknown, not custom, so it alerts. + A missing `library_name` is unknown, not custom, so it alerts - unless the repo ships its + own `modeling_*.py` and nothing calls it a Transformers model: then the class is whatever + that file defines, and without `auto_map` Transformers cannot even be told to load it. + The Hub puts `library_name: transformers` and the `transformers` tag on a repo it can + load with the library, so a new class Transformers really has keeps alerting. """ cfg = model.get("config") or {} if cfg.get("auto_map"): return True + transformers = "transformers" in (model.get("tags") or []) if model.get("library_name") in NON_TRANSFORMERS_LIBRARIES: - return "transformers" not in (model.get("tags") or []) + return not transformers + if not model.get("library_name") and not transformers: + return any(_OWN_MODELING.match((sib or {}).get("rfilename") or "") + for sib in model.get("siblings") or ()) return False @@ -299,7 +314,9 @@ def sweep_newest(limit, seen, tag="text-generation", pages_guard=20, fetch=None) """ fetch = fetch or hf_get_page fresh, batch_ids = [], set() - url = f"{API}?pipeline_tag={tag}&sort=createdAt&direction=-1&limit=100&config=true" + # full=true: the listing then carries each repo's file names (`siblings`), which is how + # is_remote_code() sees a repo's own modeling code; config=true alone leaves them out + url = f"{API}?pipeline_tag={tag}&sort=createdAt&direction=-1&limit=100&config=true&full=true" page = 0 while url and len(fresh) < limit and page < pages_guard: batch, link = fetch(url) @@ -779,7 +796,7 @@ def report(models, res, previously_seen): elif cls in quiet: owners = len({i.split("/")[0] for i in ids}) print(f" . REMOTE-CODE {cls}: {len(ids)} model(s) from {owners} uploader(s), e.g. {ids[0]}") - print(f" -> custom modeling code (auto_map, or a non-Transformers runtime); " + print(f" -> custom modeling code (auto_map, a non-Transformers runtime, or the repo's own modeling_*.py); " f"alerts once {REMOTE_CODE_SPREAD} uploaders use it") elif cls in res["significant"]: print(f" !! UNCOVERED {cls} (reviewed, not an alias — docs/arch-gaps.md): "