Repository navigation
Expand file tree
/
Copy pathgenerate.py
More file actions
1729 lines (1555 loc) · 64 KB
/
Copy pathgenerate.py
File metadata and controls
1729 lines (1555 loc) · 64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""
ModelCompass generator — pure per-benchmark rankings, no mixing.
Sources (all fetched directly, no third-party snapshots):
1. OpenRouter /api/v1/models -> catalog: pricing, context, modality, flags (deduped with models.dev)
1b. models.dev /api.json -> catalog: broad universe of "what models exist" (6667 models)
2. Artificial Analysis API v2 -> raw benchmark spine (~600 models, all written directly)
3. arena.ai (Jina Reader) -> LMArena Elo spine (11 leaderboards)
4. BenchLM (MIT JSON) -> 437 benchmarks across 388 models
Each source writes its OWN file(s) under benchmarks/. No scores are blended.
consensus/ contains a Borda-count placement-agreement summary only.
models.json carries raw benchmark cross-references, no derived scores.
"""
import json
import os
import re
import sys
import csv
import argparse
import urllib.request
import urllib.error
from datetime import datetime, timezone
try:
import yaml
except ImportError:
yaml = None
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
OR_API = "https://openrouter.ai/api/v1/models"
AA_API = "https://artificialanalysis.ai/api/v2/data/llms/models"
ARENA_BASE = "https://arena.ai/leaderboard/"
JINA_BASE = "https://r.jina.ai/"
BENCHLM_BASE = "https://benchlm.ai/data"
OUTDIR = "."
# Collapse only routing aliases; keep identity-bearing tags (e.g. -0813, -latest)
COLLAPSE_COLON = re.compile(
r"(:batch|:free|:thinking|:nitro|:floor|:cached)$"
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def log(*a):
print(*a, file=sys.stderr, flush=True)
def fetch_json(url, headers=None, timeout=60):
req = urllib.request.Request(
url, headers=headers or {"User-Agent": "modelcompass/1.0"}
)
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
def fetch_text(url, headers=None, timeout=60):
req = urllib.request.Request(
url, headers=headers or {"User-Agent": "modelcompass/1.0"}
)
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read().decode("utf-8")
def family_id(mid):
if "/" in mid:
p, r = mid.split("/", 1)
else:
p, r = "", mid
r = COLLAPSE_COLON.sub("", r)
return f"{p}/{r}" if p else r
def norm(s):
if not s:
return ""
s = s.lower()
s = re.sub(r"\d{4}-\d{2}-\d{2}", "", s)
s = re.sub(r"\([^)]*\)", "", s)
s = re.sub(r"[^a-z0-9]+", " ", s)
return s.strip()
def num(x):
try:
return float(x)
except (TypeError, ValueError):
return None
def scale(x, by=1.0):
v = num(x)
return None if v is None else v / by
# ---------------------------------------------------------------------------
# Source 1: OpenRouter catalog
# ---------------------------------------------------------------------------
def fetch_openrouter():
data = fetch_json(OR_API, timeout=60)["data"]
out = []
for m in data:
mid = m.get("id", "")
arch = m.get("architecture", {}) or {}
in_mods = set(arch.get("input_modalities", []) or [])
out_mods = set(arch.get("output_modalities", []) or [])
params = m.get("supported_parameters", []) or []
provider = mid.split("/")[0] if "/" in mid else ""
ctx = max(
m.get("context_length") or 0,
(m.get("top_provider", {}) or {}).get("context_length") or 0,
)
p_in = scale(m.get("pricing", {}).get("prompt"), 1e-6)
p_out = scale(m.get("pricing", {}).get("completion"), 1e-6)
p_cache = scale(m.get("pricing", {}).get("input_cache_read"), 1e-6)
rec = {
"id": mid,
"name": m.get("name"),
"provider": provider,
"family": family_id(mid),
"source": "openrouter",
"created_unix": m.get("created"),
"context": ctx,
"modality": arch.get("modality", ""),
"is_vision": "image" in in_mods,
"is_audio_input": "audio" in in_mods,
"is_video_input": "video" in in_mods,
"is_image_output": "image" in out_mods,
"is_audio_output": "audio" in out_mods,
"supports_reasoning": "reasoning" in params,
"supports_tools": "tools" in params,
"supports_json": ("structured_outputs" in params)
or ("response_format" in params),
"supports_caching": bool(p_cache and p_cache > 0),
"knowledge_cutoff": m.get("knowledge_cutoff"),
"hugging_face_id": m.get("hugging_face_id"),
"open_weight": bool(m.get("hugging_face_id")),
"synthetic": provider == "openrouter",
"free": bool(p_in is not None and p_in <= 0 and provider != "openrouter"),
"price_per_million": {
"prompt": round(p_in, 6) if p_in is not None else None,
"completion": round(p_out, 6) if p_out is not None else None,
"cache_read": round(p_cache, 6) if p_cache is not None else None,
},
"benchmarks": {},
}
out.append(rec)
return out
# ---------------------------------------------------------------------------
# Source 1b: models.dev catalog (broad universe of "what models exist")
# ---------------------------------------------------------------------------
MODELS_DEV_URL = "https://models.dev/api.json"
def fetch_modelsdev():
"""Return (list_of_catalog_recs, note). Deduped against OpenRouter by
base slug so routers like kilo/openrouter/... don't create dup rows.
models.dev has NO benchmark scores — it is catalog-only (pricing, context,
modality, capability flags). It widens the universe so models that exist on
AA/Arena/BenchLM but not in OpenRouter still get a catalog entry.
"""
try:
data = fetch_json(MODELS_DEV_URL, timeout=90)
except Exception as e:
return [], f"fetch error: {e}"
out = []
for prov, pdata in data.items():
for mid, mval in (pdata.get("models") or {}).items():
cost = (mval.get("cost") or {})
limit = (mval.get("limit") or {})
mods = (mval.get("modalities") or {})
in_mods = set(mods.get("input", []) or [])
out_mods = set(mods.get("output", []) or [])
p_in = scale(cost.get("input"), 1e-6)
p_out = scale(cost.get("output"), 1e-6)
rec = {
"id": mid,
"name": mval.get("name", mid),
"provider": mid.split("/")[0] if "/" in mid else prov,
"family": family_id(mid),
"source": "models.dev",
"created_unix": None,
"context": limit.get("context"),
"modality": "/".join(sorted(in_mods | out_mods)) if (in_mods | out_mods) else "",
"is_vision": "image" in in_mods,
"is_audio_input": "audio" in in_mods,
"is_video_input": "video" in in_mods,
"is_image_output": "image" in out_mods,
"is_audio_output": "audio" in out_mods,
"supports_reasoning": bool(mval.get("reasoning")),
"supports_tools": bool(mval.get("tool_call")),
"supports_json": False,
"supports_caching": False,
"knowledge_cutoff": mval.get("knowledge"),
"hugging_face_id": None,
"open_weight": bool(mval.get("open_weights")),
"synthetic": False,
"free": mid.endswith(":free"),
"price_per_million": {
"prompt": round(p_in, 6) if p_in is not None else None,
"completion": round(p_out, 6) if p_out is not None else None,
"cache_read": None,
},
"benchmarks": {},
}
out.append(rec)
return out, f"ok ({len(out)} models from {len(data)} providers)"
# ---------------------------------------------------------------------------
# Source 1c: OpenRouter usage rankings (real-world adoption signal)
# ---------------------------------------------------------------------------
OR_RANKINGS_URL = "https://openrouter.ai/api/frontend/v1/rankings/models"
def fetch_or_usage():
"""Return (usage_dict, note). usage_dict maps model_permaslug -> aggregate
usage over the returned window: total tokens (prompt+completion), request
count. This is REAL-WORLD USAGE, not a quality benchmark — it lives in its
own spine and never blends into scores."""
try:
data = fetch_json(OR_RANKINGS_URL, timeout=60)["data"]
except Exception as e:
return {}, f"fetch error: {e}"
agg = {}
for row in data:
slug = row.get("model_permaslug") or ""
if not slug:
continue
a = agg.setdefault(slug, {"tokens": 0, "requests": 0, "days": set()})
a["tokens"] += (row.get("total_prompt_tokens") or 0) + (
row.get("total_completion_tokens") or 0
)
a["requests"] += row.get("count") or 0
if row.get("date"):
a["days"].add(row["date"][:10])
for a in agg.values():
a.pop("days", None)
return agg, f"ok ({len(agg)} models with live usage)"
# ---------------------------------------------------------------------------
# Source 1d: DeepSWE (datacurve.ai) — agentic software-engineering eval
# ---------------------------------------------------------------------------
DEEPSWE_URL = (
"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"
)
def fetch_deepswe():
"""Return (rows, note). DeepSWE = pass@1 on 113 real GitHub issues run
through agent harnesses (mini-swe-agent etc). Fresh artifact JSON with
generated_at; soft-fail with note if unavailable."""
try:
d = fetch_json(DEEPSWE_URL, timeout=60)
except Exception as e:
return [], f"fetch error: {e}"
rows = d.get("rows") or []
out = []
for r in rows:
if not r.get("model"):
continue
out.append({
"model": r["model"],
"harness": r.get("harness"),
"reasoning_effort": r.get("reasoning_effort"),
"pass_rate": r.get("pass_rate"),
"n_passed": r.get("n_passed"),
"n_attempted": r.get("n_attempted"),
"mean_cost_usd": r.get("mean_cost_usd"),
"mean_output_tokens": r.get("mean_output_tokens"),
})
return out, f"ok ({len(out)} configs, generated_at={d.get('generated_at', '?')[:10]})"
# ---------------------------------------------------------------------------
# Source 2: Artificial Analysis
# ---------------------------------------------------------------------------
def fetch_artificial_analysis(api_key):
if not api_key:
return None, "no API key"
req = urllib.request.Request(
AA_API,
headers={"User-Agent": "modelcompass/1.0", "x-api-key": api_key},
)
try:
with urllib.request.urlopen(req, timeout=60) as r:
payload = json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
return None, f"HTTP {e.code}: {e.reason}"
except Exception as e:
return None, str(e)
models = payload.get("data", []) if isinstance(payload, dict) else []
out = {}
for m in models:
creator = (m.get("model_creator") or {}).get("slug") or ""
slug = m.get("slug") or ""
evals = m.get("evaluations") or {}
pricing = m.get("pricing") or {}
key = f"{creator}/{slug}".lower()
out[key] = {
"key": key,
"name": m.get("name"),
"creator_slug": creator,
"slug": slug,
"evaluations": evals,
"pricing": {
"prompt": scale(pricing.get("price_1m_input_tokens")),
"completion": scale(pricing.get("price_1m_output_tokens")),
"blended": scale(pricing.get("price_1m_blended_3_to_1")),
},
"median_tps": num(m.get("median_output_tokens_per_second")),
"ttft": num(m.get("median_time_to_first_token_seconds")),
}
return out, f"ok ({len(out)} models)"
# ---------------------------------------------------------------------------
# Source 4: arena.ai (Jina Reader — same technique as community scrapers)
# ---------------------------------------------------------------------------
def _parse_number(s):
if not s:
return None
s = s.strip()
m = re.match(r"([\d.]+)", s)
if not m:
return None
try:
return float(m.group(1))
except ValueError:
return None
def _parse_number_with_ci(s):
"""'12.19%±1.45%' -> (12.19, 1.45) ; '12.19%' -> (12.19, None)."""
if not s:
return None, None
s = s.strip()
score = None
ci = None
m = re.match(r"([\d.]+)\s*%?", s)
if m:
score = float(m.group(1))
m_ci = re.search(r"±\s*([\d.]+)\s*%?", s)
if m_ci:
ci = float(m_ci.group(1))
return score, ci
def _parse_int(s):
if not s:
return None
# keep only digits / commas (drops trailing labels like "votes")
digits = re.sub(r"[^0-9,]", "", s.strip())
digits = digits.replace(",", "")
if not digits:
return None
try:
return int(digits)
except ValueError:
return None
def _is_sep_row(line):
"""True for a markdown table separator row like |---|---:---|."""
s = line.strip()
if not s.startswith("|"):
return False
cells = [c.strip() for c in s.strip("|").split("|")]
return bool(cells) and all(re.fullmatch(r":?-+:?", c) for c in cells if c != "")
def _parse_arena_general(content, slug):
"""Parse a general arena.ai leaderboard (text, code, vision, …).
Change-proof against Arena's display format:
* Supports the current cell-per-line layout (each field on its own line,
tab/newline separated) AND legacy markdown `|` tables.
* Records are anchored on the VENDOR line (contains '·'), not on a column
header text, so renaming/reordering columns never hides the data.
* Score / CI / votes are located relative to the vendor line by position,
with `±` detection for CI, so inserted/renamed columns can't shift values.
"""
lines = content.split("\n")
# ---- Path A: current cell-per-line layout (each field on its own line) ----
models_a = _parse_arena_cell_layout(lines, slug)
if models_a:
return {
"meta": {
"leaderboard": slug,
"source_url": f"https://arena.ai/leaderboard/{slug}",
"fetched_at": datetime.now(timezone.utc).isoformat(),
"model_count": len(models_a),
"layout": "cell",
},
"models": models_a,
}, f"ok ({len(models_a)} models)"
# ---- Path B: legacy markdown `|` table layout ----
models = []
header_cells = None
pending_header = None
in_table = False
for line in lines:
if line.startswith("|"):
if _is_sep_row(line):
# Header = the | row immediately before this separator.
header_cells = pending_header
in_table = True
pending_header = None
continue
pending_header = line
if not in_table or header_cells is None:
continue
else:
# Non-table line resets the pending header so later tables still parse.
pending_header = None
continue
cells = [c.strip() for c in line.strip("|").split("|")]
if len(cells) < 3:
continue
rank_m = re.match(r"\s*(\d+)", cells[0])
if not rank_m:
continue
rank = int(rank_m.group(1))
# Model cell = first cell that is a markdown link [name](http...)
model_idx = None
for i, c in enumerate(cells[1:], start=1):
if re.search(r"\[[^\]]+\]\(https?://", c):
model_idx = i
break
if model_idx is None:
continue
m = re.match(r"\[([^\]]+)\]", cells[model_idx])
model_name = m.group(1) if m else cells[model_idx]
vendor_match = re.search(r"\]\([^)]*\)\s*([^·]+?)\s*·", cells[model_idx])
vendor = vendor_match.group(1).strip() if vendor_match else None
lic_match = re.search(
r"·\s*(proprietary|open|Open Source|MIT|Apache|GPL|CC-|Community|Non-commercial)",
cells[model_idx], re.I,
)
license = "proprietary" if (
lic_match and "proprietary" in lic_match.group(1).lower()
) else ("open" if lic_match else None)
# Score / CI / votes by role (not fixed offset)
tail = cells[model_idx + 1:]
score = None
ci = None
votes = None
for c in tail:
if score is None:
val = _parse_number(c)
if val is not None:
score = val
ci_m = re.search(r"±\s*[\d.]+", c)
ci = _parse_number(ci_m.group(0)) if ci_m else None
continue
if ci is None and "±" in c:
ci = _parse_number(c)
continue
if votes is None and re.search(r"votes", c, re.I):
votes = _parse_int(c)
models.append(
{
"rank": rank,
"model": model_name,
"vendor": vendor,
"license": license,
"score": score,
"ci": ci,
"votes": votes,
}
)
return {
"meta": {
"leaderboard": slug,
"source_url": f"https://arena.ai/leaderboard/{slug}",
"fetched_at": datetime.now(timezone.utc).isoformat(),
"model_count": len(models),
"layout": "table",
},
"models": models,
}, f"ok ({len(models)} models)"
def _parse_arena_cell_layout(lines, slug):
"""Parse the current Arena layout where each field is on its own line.
A record looks like:
<rank>
<previous rank> (or blank)
<badge / extra int> (optional)
<model slug> (may contain spaces/parens, e.g. 'muse-spark-1.2 (xHigh)')
<Vendor> · <License>
<score>
±<ci>
<votes>\\t<price>\\t<context>
We anchor on the vendor line (contains '·') and read the surrounding fields
by position, so renamed/reordered columns cannot hide the data.
"""
models = []
n = len(lines)
i = 0
while i < n:
line = lines[i].strip()
# Vendor line marks a record; model slug is the line immediately above it.
if "·" in line and i > 0:
vendor_line = line
model = lines[i - 1].strip()
# Score = first numeric line after the vendor line
j = i + 1
while j < n and lines[j].strip() == "":
j += 1
score = _parse_number(lines[j]) if j < n else None
ci = None
if j + 1 < n and lines[j + 1].strip().startswith("±"):
ci = _parse_number(re.sub(r"^±\s*", "", lines[j + 1]))
# Combined votes/price/context line
k = j + 2
while k < n and lines[k].strip() == "":
k += 1
combined = lines[k].strip() if k < n else ""
votes = None
if combined:
# votes is the first integer token in the combined line
vm = re.search(r"(\d[\d,]*)", combined)
if vm:
votes = _parse_int(vm.group(1))
# Rank = the FARTHEST-back pure-integer line in the window above the
# vendor line (record order is: rank, prev-rank, badge, model, vendor).
# Scan from farthest (back=6) toward the vendor so we take the first
# one in document order, which is the true rank.
rank = None
for back in range(6, 1, -1):
cand = lines[i - back].strip() if i - back >= 0 else ""
if re.fullmatch(r"\d+", cand):
rank = int(cand)
break
vendor_match = re.search(r"([^·]+?)\s*·", vendor_line)
vendor = vendor_match.group(1).strip() if vendor_match else None
lic_match = re.search(
r"·\s*(proprietary|open|Open Source|MIT|Apache|GPL|CC-|Community|Non-commercial)",
vendor_line, re.I,
)
license = "proprietary" if (
lic_match and "proprietary" in lic_match.group(1).lower()
) else ("open" if lic_match else None)
if model and score is not None:
models.append(
{
"rank": rank if rank is not None else len(models) + 1,
"model": model,
"vendor": vendor,
"license": license,
"score": score,
"ci": ci,
"votes": votes,
}
)
i = k + 1
continue
i += 1
return models
def _parse_arena_agent(content, slug):
"""Parse the agent leaderboard (dimension scores per model).
Layout-change-proof:
* Table detected by its separator row, not the header text.
* The model cell is the first markdown-link cell in the header; dimension
names are the cells between the model cell and the end (minus known
trailing columns like Sessions/Price), so renaming Rank/Model/Sessions
or inserting columns does not break dimension mapping.
"""
lines = content.split("\n")
models = []
dimensions = []
model_col = 1
header_cells = None
pending_header = None
in_table = False
for line in lines:
if line.startswith("|"):
if _is_sep_row(line):
# Header = the | row immediately before this separator.
header_cells = pending_header
in_table = True
pending_header = None
if header_cells is not None:
hc = [c.strip() for c in header_cells.strip("|").split("|")]
for i, c in enumerate(hc):
if re.search(r"\[[^\]]+\]\(https?://", c):
model_col = i
break
trailing = {"Sessions", "Price $/M", "Price $/M tokens", "Price"}
dims = []
for c in hc[model_col + 1:]:
name = re.sub(r"\s*\([^)]*\)", "", c).strip()
if not name or name in trailing:
continue
dims.append(name)
dimensions = dims
continue
pending_header = line
if not in_table or header_cells is None:
continue
else:
pending_header = None
continue
cells = [c.strip() for c in line.strip("|").split("|")]
if len(cells) < 3:
continue
rank_m = re.match(r"\s*(\d+)", cells[0])
if not rank_m:
continue
rank = int(rank_m.group(1))
model_cell = cells[model_col] if model_col < len(cells) else (cells[1] if len(cells) > 1 else "")
m = re.match(r"\[([^\]]+)\]", model_cell)
model_name = m.group(1) if m else model_cell
vendor_match = re.search(r"\]\([^)]*\)\s*([^·]+?)\s*·", model_cell)
vendor = vendor_match.group(1).strip() if vendor_match else None
lic_match = re.search(
r"·\s*(proprietary|open|Open Source|MIT|Apache|GPL|CC-|Community|Non-commercial)",
model_cell, re.I,
)
license = "proprietary" if (
lic_match and "proprietary" in lic_match.group(1).lower()
) else ("open" if lic_match else None)
# Dimension scores are the cells immediately after the model cell.
scores = []
for i, dim in enumerate(dimensions):
cell_idx = model_col + 1 + i
if cell_idx < len(cells):
score_val, ci_val = _parse_number_with_ci(cells[cell_idx])
scores.append({"name": dim, "score": score_val, "ci": ci_val})
# Sessions = last numeric-ish cell before any trailing price column.
sessions = _parse_int(cells[-1]) if cells else None
models.append(
{
"rank": rank,
"model": model_name,
"vendor": vendor,
"license": license,
"scores": scores,
"sessions": sessions,
}
)
return {
"meta": {
"leaderboard": slug,
"source_url": f"https://arena.ai/leaderboard/{slug}",
"fetched_at": datetime.now(timezone.utc).isoformat(),
"model_count": len(models),
"dimensions": dimensions,
"layout": "table",
},
"models": models,
}, f"ok ({len(models)} models)"
def fetch_arena_leaderboard(slug):
"""Fetch a single arena.ai leaderboard via Jina Reader."""
url = f"{JINA_BASE}{ARENA_BASE}{slug}"
req = urllib.request.Request(
url,
headers={
"Accept": "application/json",
"X-Return-Format": "markdown",
"User-Agent": "modelcompass/1.0",
},
)
try:
with urllib.request.urlopen(req, timeout=60) as r:
wrapper = json.loads(r.read().decode("utf-8"))
except Exception as e:
return None, f"fetch error: {e}"
content = wrapper.get("data", {}).get("content", "")
if not content:
return None, "empty content"
try:
if slug == "agent":
return _parse_arena_agent(content, slug)
return _parse_arena_general(content, slug)
except Exception as e:
# A layout change on this board must never crash the whole run.
return None, f"parse error (board layout changed?): {e}"
# Known arena.ai leaderboards (fetched live from each page — not a third-party
# snapshot). The overview page is a JS SPA and no longer exposes stable links,
# so we enumerate the well-known boards here; the discover scrape is a fallback.
ARENA_SLUGS = [
"agent",
"code",
"text",
"vision",
"document",
"search",
"image-edit",
"image-to-video",
"text-to-image",
"text-to-video",
"video-edit",
]
def discover_arena_slugs():
"""Return the set of arena.ai leaderboard slugs to fetch.
Strategy (defense in depth, NOT purely hardcoded):
1. Scrape the live overview page via Jina Reader first. This auto-discovers
any NEW board arena.ai adds, so we pick up changes without code edits.
2. Union with a curated fallback list (ARENA_SLUGS) so that if the overview
page's format changes / the scrape returns nothing, we never silently
regress to zero boards.
"""
scraped = set()
try:
text = fetch_text(f"{JINA_BASE}{ARENA_BASE}")
scraped.update(re.findall(r"arena\.ai/leaderboard/([a-z][a-z0-9-]*)", text))
except Exception as e:
log(f"Arena slug scrape failed ({e}); using curated fallback")
if scraped:
log(f"Arena slug scrape found {len(scraped)} boards (live)")
else:
log("Arena slug scrape found 0 — using curated fallback list")
new_boards = sorted(scraped - set(ARENA_SLUGS))
if new_boards:
log(f"Arena: {len(new_boards)} new board(s) not in curated list: {new_boards}")
slugs = set(ARENA_SLUGS) | scraped
return sorted(slugs), sorted(scraped)
# ---------------------------------------------------------------------------
# Source 5: BenchLM (MIT licensed raw JSON)
# ---------------------------------------------------------------------------
def fetch_benchlm():
"""Fetch BenchLM models.json — 388 models, 437 benchmarks."""
try:
data = fetch_json(f"{BENCHLM_BASE}/models.json", timeout=60)
except Exception as e:
return None, str(e)
items = data.get("items", []) if isinstance(data, dict) else []
return items, f"ok ({len(items)} models)"
# ---------------------------------------------------------------------------
# Merge: cross-reference only, no score blending
# ---------------------------------------------------------------------------
def base_slug(model_id):
"""Strip reasoning-effort / deviation suffixes (xhigh, high, max, xlow,
low, medium, batch, free, latest, thinking, -0824, -0813, -3107 …) so that
e.g. 'openai/gpt-5.6-sol-high' collapses to 'openai/gpt-5.6-sol'.
Composed prefixes like 'deepseek/deepseek-v4-pro-0813' have NO base and
return as-is on purpose."""
parts = model_id.rsplit("-", 1)
if len(parts) != 2:
return model_id
head, tail = parts
# suffix tokens that always mean 'a variant of the same model'
if re.fullmatch(r"(x?high|x?low|medium|batch|free|latest|thinking|"
r"\d{3,4}|"
r"[a-z]{1,4}\d{1,4})", tail):
return head
return model_id
def merge(catalog, aa, arena, benchlm):
"""Attach catalog enrichment to benchmark spines (no filtering).
Each benchmark stays in its own namespace under benchmarks/<source>/.
`catalog` = OR ∪ models.dev universe. We use it only to DECORATE
benchmark rows with pricing/context/flags where a match is found — never
to decide whether a benchmark row exists.
"""
cat_by_id = {}
cat_by_norm = {}
cat_by_slug_only = {}
for m in catalog:
mid = m["id"].lower()
cat_by_id[mid] = m
cat_by_norm.setdefault(norm(m["name"]), []).append(m)
slug_only = mid.split("/", 1)[-1] if "/" in mid else mid
cat_by_slug_only.setdefault(slug_only, []).append(m)
counts = {"aa": 0, "arena": 0, "benchlm": 0}
# Attach AA
if aa:
for key, a in aa.items():
creator = a.get("creator_slug", "")
slug = a.get("slug", "")
target = None
k = f"{creator}/{slug}".lower()
if k in cat_by_id:
target = cat_by_id[k]
else:
base = base_slug(k.replace(".", "-"))
if base in cat_by_id:
target = cat_by_id[base]
else:
so = base.split("/", 1)[-1]
if so in cat_by_slug_only:
target = cat_by_slug_only[so][0]
else:
nn = norm(a.get("name", ""))
if nn in cat_by_norm:
target = cat_by_norm[nn][0]
if target:
target["benchmarks"]["aa"] = a
counts["aa"] += 1
# Attach arena
if arena:
for lb_slug, lb_data in arena.items():
for model_info in lb_data.get("models", []):
nn = norm(model_info.get("model", ""))
if nn in cat_by_norm:
for m in cat_by_norm[nn]:
m["benchmarks"].setdefault("arena", {})
m["benchmarks"]["arena"][lb_slug] = model_info
counts["arena"] += 1
break
# Attach BenchLM
if benchlm:
for bm in benchlm:
slug = bm.get("slug", "").lower()
model_name = bm.get("model", "")
target = None
if slug and slug in cat_by_id:
target = cat_by_id[slug]
else:
if slug in cat_by_slug_only:
target = cat_by_slug_only[slug][0]
elif slug:
base = base_slug(slug)
if base in cat_by_slug_only:
target = cat_by_slug_only[base][0]
if target is None:
nn = norm(model_name)
if nn in cat_by_norm:
target = cat_by_norm[nn][0]
if target:
target["benchmarks"]["benchlm"] = bm
counts["benchlm"] += 1
return counts
# ---------------------------------------------------------------------------
# Output: per-benchmark files
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Source registry: where each benchmark lives + how to present it
# ---------------------------------------------------------------------------
SOURCE_META = {
"artificial-analysis": {
"title": "Artificial Analysis",
"source_url": "https://artificialanalysis.ai",
"desc": "Raw benchmark evaluations from Artificial Analysis' proprietary "
"suite (AA API v2). Each file is one benchmark — higher is better "
"unless noted. Values are the model's raw score on that benchmark.",
"benches": {
"aa_intelligence": ("Intelligence Index", "Composite intelligence index (0–100)."),
"aa_coding": ("Coding Index", "Composite coding capability index (0–100)."),
"aa_math": ("Math Index", "Composite math reasoning index (0–100)."),
"aa_aime": ("AIME", "American Invitational Mathematics Examination accuracy (%)."),
"aa_aime_25": ("AIME 2025", "AIME 2025 accuracy (%)."),
"aa_gpqa": ("GPQA", "Graduate-level Google-Proof Q&A diamond accuracy (%)."),
"aa_hle": ("HLE", "Humanity's Last Exam accuracy (%)."),
"aa_ifbench": ("IFBench", "Instruction-following benchmark score (%)."),
"aa_lcr": ("LCR", "Long-context reasoning score (%)."),
"aa_livecodebench": ("LiveCodeBench", "LiveCodeBench coding accuracy (%)."),
"aa_math_500": ("MATH-500", "MATH-500 competition-math accuracy (%)."),
"aa_mmlu_pro": ("MMLU-Pro", "MMLU-Pro knowledge accuracy (%)."),
"aa_scicode": ("SciCode", "SciCode scientific coding accuracy (%)."),
"aa_tau2": ("TAU2", "TAU-bench agent benchmark (%)."),
"aa_tau_banking": ("TAU-Banking", "TAU-bench banking domain agent score (%)."),
"aa_terminalbench_hard": ("Terminal-Bench Hard", "Terminal-Bench hard sys-admin tasks (%)."),
"aa_terminalbench_v2_1": ("Terminal-Bench v2.1", "Terminal-Bench v2.1 sys-admin tasks (%)."),
},
},
"arena": {
"title": "LMArena (arena.ai)",
"source_url": "https://arena.ai",
"desc": "Human-preference Elo from blind pairwise battles. "
"Higher Elo = more preferred by human voters.",
"benches": {}, # filled dynamically from discovered slugs
},
"benchlm": {
"title": "BenchLM",
"source_url": f"{BENCHLM_BASE}/models.json",
"desc": "Aggregated category scores (0–100) across 437 benchmarks and 388 "
"models (MIT-licensed). Higher is better.",
"benches": {
"benchlm_agentic": ("Agentic", "Tool-use / agentic task category score."),
"benchlm_coding": ("Coding", "Code generation category score."),
"benchlm_math": ("Math", "Quantitative reasoning category score."),
"benchlm_reasoning": ("Reasoning", "Logical reasoning category score."),
"benchlm_knowledge": ("Knowledge", "Factual knowledge category score."),
"benchlm_multilingual": ("Multilingual", "Non-English capability category score."),
"benchlm_multimodalGrounded": ("Multimodal Grounded", "Vision+text grounded understanding."),
"benchlm_instructionFollowing": ("Instruction Following", "Following complex instructions."),
},
},
"consensus": {
"title": "Consensus (Borda)",
"source_url": "",
"desc": "Placement agreement across all benchmarks. Counts how many top-10 "
"lists a model appears in — never an averaged score.",
"benches": {
"consensus": ("Consensus Ranking", "Top-10 appearance count, tie-broken by Borda points."),
},
},
"openrouter-usage": {
"title": "OpenRouter Usage",
"source_url": OR_RANKINGS_URL,
"desc": "Real-world adoption: tokens processed and request counts via "
"OpenRouter (rolling window). NOT a quality score — usage is "
"never blended into benchmark rankings.",
"benches": {
"or_usage_tokens": ("Usage Rank", "Rank by total tokens processed; requests kept alongside."),
},
},
"deepswe": {
"title": "DeepSWE (Datacurve)",
"source_url": DEEPSWE_URL,
"desc": "Agentic software-engineering eval: pass@1 on 113 real GitHub "
"issues executed through agent harnesses. Cost and token stats "
"kept as raw columns, never blended.",
"benches": {
"deepswe_v1_1": ("DeepSWE v1.1", "pass@1 per harness+model+effort config."),
},
},
}
WRITTEN = {} # subdir -> [bname, ...]
def _write_benchmark_json(subdir, bname, meta_info, models_list):
out_dir = os.path.join("benchmarks", subdir)
os.makedirs(out_dir, exist_ok=True)
data = {"meta": meta_info, "models": models_list}
path = os.path.join(out_dir, f"{bname}.json")
with open(path, "w") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
WRITTEN.setdefault(subdir, []).append(bname)
return path
def write_aa_benchmarks(aa):
"""Write every Artificial Analysis benchmark into artificial-analysis/.
AA is its OWN spine: we iterate the full AA response (every model AA
returns), NOT just OpenRouter-matched models. This recovers models like
meituan/longcat-2.0 that exist on AA but not in the OR catalog.
We emit one file per evaluation key that actually appears across the AA
catalog (17 fields), sorted by score (higher = better). `fetched_at` is
stamped on each file's meta.
"""
subdir = "artificial-analysis"
if not aa:
return {}
fetched_at = datetime.now(timezone.utc).isoformat()
# Discover which evaluation keys exist across the full AA catalog
seen = {}
for key, a in aa.items():
evals = (a.get("evaluations") or {})