From 05e5ad782dd9bc6aa491bde8e6aa060de59d58dd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 03:41:18 -0400 Subject: [PATCH 1/3] tools/ci/notice_covers_deps.py: a dependency compiled into the binary must be named in NOTICE Found reviewing #2335: it added stb_image -- fetched by FetchContent, compiled into the app, shipped in the binary -- and NOTICE, the file that exists to record exactly that, was not in the diff. Nothing looked, because NOTICE is maintained by hand. notice_covers_deps.py --check every FetchContent_Declare's GIT_REPOSITORY appears in NOTICE notice_covers_deps.py --self-test 10 arms It matches on the URL rather than the name, because the URL is what the two files share: CMake calls it `stb`, NOTICE calls it `stb_image (nothings/stb)`, and a name match would miss the very case that prompted this. A declaration with no URL is not this check's finding; a CMakeLists with no declarations is 'cannot tell', rc 2, not a pass. On this tree: 3 of 3 named, rc 0. Strip stb_image's NOTICE block and it is rc 1 naming 'stb (https://github.com/nothings/stb.git) is fetched and not named in NOTICE' -- the case that happened, reproduced. Registered as two ctest rows (self-test + the check); both run offline. --- CMakeLists.txt | 6 ++ tools/ci/notice_covers_deps.py | 154 +++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100755 tools/ci/notice_covers_deps.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dbee52ac6..01fe07749 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -398,6 +398,12 @@ add_python_test(moe_expert_chunks_accum_test ${CMAKE_SOURCE_DIR}/tests/moe_exper # engine#2138 slice: a merge's conflicts classified by kind, from git's index -- the conflicts a text search cannot # see (add/delete with no markers) are the ones a pin merge is mostly made of. Device-free, no checkout. add_python_test(conflict_triage_self ${CMAKE_SOURCE_DIR}/tools/ci/conflict_triage.py --self-test) +# engine#2335 review: a FetchContent dependency compiled into the app with no NOTICE entry. NOTICE is +# maintained by hand and nothing compared it to CMakeLists, so `stb_image` was fetched, compiled and +# shipped without a licence line until a reviewer noticed. Matched on the GIT_REPOSITORY URL: that is +# what the two files share (`stb` in CMake is `stb_image (nothings/stb)` in NOTICE, so the name is not). +add_python_test(notice_covers_deps_self ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_deps.py --self-test) +add_python_test(notice_covers_deps ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_deps.py --check) # reviews: a PR that carries another PR's files lands two merge decisions as one, and one whose work base # already has reads as a fresh duplicate. Both are invisible to a diffstat; this compares blobs. Device-free. add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check.py --self-test) diff --git a/tools/ci/notice_covers_deps.py b/tools/ci/notice_covers_deps.py new file mode 100755 index 000000000..9e366cccb --- /dev/null +++ b/tools/ci/notice_covers_deps.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""notice_covers_deps.py -- every FetchContent dependency has a NOTICE entry. + +`NOTICE` is the file that says what is compiled into this binary and under which licence. It is +maintained by hand, and on 2026-11-13 a PR added `stb_image` -- fetched, compiled into the app, shipped +in the binary -- with no entry: nothing looked. This asks the one question that finding needed, and it +reads the two files that already exist rather than a registry someone has to remember to update. + + notice_covers_deps.py [--check] [--root ROOT] + +Matches on the `GIT_REPOSITORY` URL, not on the dependency name: `FetchContent_Declare(stb ...)` is +called `stb` in CMake and appears in NOTICE as `stb_image (nothings/stb ...)`, so the name is not the +thing the two files share -- the URL is. Exit 0 when every dependency is covered, 1 when one is not, +2 on a usage error. +""" + +import argparse +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile + +DECLARE = re.compile(r"FetchContent_Declare\(\s*([A-Za-z0-9_.+-]+)((?:[^()]|\([^()]*\))*)\)", re.S) +URL = re.compile(r"(?:GIT_REPOSITORY|URL)\s+(\S+)") + + +def deps(text): + """[(name, url)] for every FetchContent_Declare in the CMake text, in order.""" + out = [] + for m in DECLARE.finditer(text): + name, body = m.group(1), m.group(2) + u = URL.search(body) + out.append((name, u.group(1) if u else "")) + return out + + +def covered(url, notice): + """True when NOTICE names this dependency -- by its URL's host/path, or its full URL. + + The URL is what the two files share, so it is what this compares; a bare repository name would + match prose (`stb` appears in sentences about the image decoder) and a name mismatch (`stb` in + CMake vs `stb_image` in NOTICE) is exactly the case that was missed. + """ + if not url: + return True # no URL to compare (an in-source or archive dep): not this check's finding + u = url.strip().rstrip("/") + short = re.sub(r"^[a-z+]+://", "", u) + short = re.sub(r"\.git$", "", short) + return short in notice + + +def check(root): + root = pathlib.Path(root) + cmake = root / "CMakeLists.txt" + notice_path = root / "NOTICE" + if not cmake.is_file(): + print(f"notice_covers_deps: no CMakeLists.txt under {root}", file=sys.stderr) + return 2 + if not notice_path.is_file(): + print(f"notice_covers_deps: no NOTICE under {root} -- a dependency list with no licence file", + file=sys.stderr) + return 2 + notice = notice_path.read_text(errors="replace") + ds = deps(cmake.read_text(errors="replace")) + if not ds: + print("notice_covers_deps: no FetchContent_Declare in CMakeLists.txt (cannot tell, and that is " + "not a pass)", file=sys.stderr) + return 2 + missing = [(n, u) for n, u in ds if not covered(u, notice)] + for n, u in missing: + print(f"notice_covers_deps: {n} ({u or 'no URL'}) is fetched and not named in NOTICE") + print(f"notice_covers_deps: {len(ds) - len(missing)} of {len(ds)} FetchContent dependency(ies) named in NOTICE") + return 1 if missing else 0 + + +def self_test(): + failures = 0 + + def chk(ok, what): + nonlocal failures + print(("ok " if ok else "FAIL ") + what) + if not ok: + failures += 1 + + cm = ("FetchContent_Declare(json\n GIT_REPOSITORY https://github.com/nlohmann/json.git\n GIT_TAG v3.11.3)\n" + "FetchContent_Declare(stb\n GIT_REPOSITORY https://github.com/nothings/stb.git)\n") + d = deps(cm) + chk(len(d) == 2 and d[0] == ("json", "https://github.com/nlohmann/json.git"), + "both declarations are read, with their repository URL") + notice = "JSON for Modern C++ (nlohmann/json v3.11.3)\n https://github.com/nlohmann/json\n" + chk(covered(d[0][1], notice), "a URL named in NOTICE counts") + chk(not covered(d[1][1], notice), "a URL absent from NOTICE does not") + chk(covered("", notice), "a declaration with no URL is not this check's finding") + chk(covered(d[0][1] + "/", notice), "a trailing slash on the URL does not change the answer") + chk(not covered("https://example.com/stb.git", "https://example.com/not-stb\n"), + "a substring of a longer path is not a match") + + # end to end on a scratch tree: one dep covered, one not -> rc 1 and the uncovered one named + tmp = tempfile.mkdtemp(prefix="notice-deps-") + try: + (pathlib.Path(tmp) / "CMakeLists.txt").write_text(cm) + (pathlib.Path(tmp) / "NOTICE").write_text(notice) + out = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], + capture_output=True, text=True) + chk(out.returncode == 1, "a tree with one uncovered dependency is rc 1") + chk("stb" in out.stdout and "not named in NOTICE" in out.stdout, + "and the message names the dependency") + (pathlib.Path(tmp) / "NOTICE").write_text(notice + "stb_image (nothings/stb)\n https://github.com/nothings/stb\n") + out2 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], + capture_output=True, text=True) + chk(out2.returncode == 0, "and covering it makes the tree rc 0") + (pathlib.Path(tmp) / "CMakeLists.txt").write_text("project(x)\n") + out3 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], + capture_output=True, text=True) + chk(out3.returncode == 2, "a CMakeLists with no declarations is 'cannot tell', rc 2") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + print(f"notice_covers_deps --self-test: {'FAIL' if failures else 'ok'}") + return 1 if failures else 0 + + +def main(argv): + ap = argparse.ArgumentParser(description="every FetchContent dependency has a NOTICE entry") + ap.add_argument("--check", action="store_true") + ap.add_argument("--root", default=".") + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args(argv) + if a.self_test: + return self_test() + if not a.check: + ap.print_usage(sys.stderr) + return 2 + return check(a.root) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 24b58e75a1629dcad9c2e43bbe8ba321b07d20e7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 04:11:25 -0400 Subject: [PATCH 2/3] notice_covers_deps: cover submodules too, not only FetchContent The first version asked the question for FetchContent dependencies only, which is the case that prompted it (#2335 added stb_image) but not the bigger half: seven submodules are compiled into this binary -- third_party/llama.cpp is the largest thing in it -- and a submodule with no NOTICE entry is the same defect, and the one a reader is least likely to notice because the code is already sitting under third_party/. Matched on the .gitmodules path, because that is how NOTICE names them ("llama.cpp (third_party/llama.cpp: 1bit-MONSTER/llama.cpp, a fork of ...)"). On this tree: 3 of 3 FetchContent and 7 of 7 submodules named, rc 0. Removing the xdna-driver NOTICE line gives rc 1 naming 'submodule third_party/xdna-driver ... is compiled in and not named in NOTICE'. 13 arms now, including a submodule with no entry -> rc 1 (and the path named), and naming the path -> rc 0. --- tools/ci/notice_covers_deps.py | 57 ++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/tools/ci/notice_covers_deps.py b/tools/ci/notice_covers_deps.py index 9e366cccb..8afb1b9b3 100755 --- a/tools/ci/notice_covers_deps.py +++ b/tools/ci/notice_covers_deps.py @@ -22,10 +22,14 @@ notice_covers_deps.py [--check] [--root ROOT] -Matches on the `GIT_REPOSITORY` URL, not on the dependency name: `FetchContent_Declare(stb ...)` is -called `stb` in CMake and appears in NOTICE as `stb_image (nothings/stb ...)`, so the name is not the -thing the two files share -- the URL is. Exit 0 when every dependency is covered, 1 when one is not, -2 on a usage error. +Two kinds of dependency are compiled in and both have to be named. A **FetchContent** one is matched on its +`GIT_REPOSITORY` URL, not its name: `FetchContent_Declare(stb ...)` is `stb` in CMake and +`stb_image (nothings/stb ...)` in NOTICE, so the URL is what the two files share. A **submodule** is matched +on its `path` from `.gitmodules` (`third_party/llama.cpp`, the largest thing in the binary, is named that +way). A submodule with no entry is the same defect and the one a reader is least likely to notice, because +the code is already sitting under `third_party/`. + +Exit 0 when every dependency is covered, 1 when one is not, 2 on a usage error. """ import argparse @@ -65,6 +69,27 @@ def covered(url, notice): return short in notice +def submodules(root): + """[(name, path)] from .gitmodules -- the second kind of compiled-in dependency.""" + gm = root / ".gitmodules" + if not gm.is_file(): + return [] + out, name, path = [], "", "" + for line in gm.read_text(errors="replace").splitlines(): + m = re.match(r'\[submodule "([^"]+)"\]', line.strip()) + if m: + if name and path: + out.append((name, path)) + name, path = m.group(1), "" + continue + kv = re.match(r"\s*path\s*=\s*(\S+)", line) + if kv: + path = kv.group(1) + if name and path: + out.append((name, path)) + return out + + def check(root): root = pathlib.Path(root) cmake = root / "CMakeLists.txt" @@ -85,8 +110,13 @@ def check(root): missing = [(n, u) for n, u in ds if not covered(u, notice)] for n, u in missing: print(f"notice_covers_deps: {n} ({u or 'no URL'}) is fetched and not named in NOTICE") - print(f"notice_covers_deps: {len(ds) - len(missing)} of {len(ds)} FetchContent dependency(ies) named in NOTICE") - return 1 if missing else 0 + subs = submodules(root) + subs_missing = [(n, p) for n, p in subs if p not in notice] + for n, p in subs_missing: + print(f"notice_covers_deps: submodule {p} ({n}) is compiled in and not named in NOTICE") + print(f"notice_covers_deps: {len(ds) - len(missing)} of {len(ds)} FetchContent dependency(ies) and " + f"{len(subs) - len(subs_missing)} of {len(subs)} submodule(s) named in NOTICE") + return 1 if (missing or subs_missing) else 0 def self_test(): @@ -125,6 +155,21 @@ def chk(ok, what): out2 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], capture_output=True, text=True) chk(out2.returncode == 0, "and covering it makes the tree rc 0") + # the second kind: a submodule compiled in with no NOTICE entry + (pathlib.Path(tmp) / "CMakeLists.txt").write_text(cm) + (pathlib.Path(tmp) / ".gitmodules").write_text( + '[submodule "third_party/x"]\n\tpath = third_party/x\n\turl = https://example.com/x.git\n') + out4 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], + capture_output=True, text=True) + chk(out4.returncode == 1, "a submodule with no NOTICE entry is rc 1") + chk("third_party/x" in out4.stdout, "and the message names the path") + (pathlib.Path(tmp) / "NOTICE").write_text( + notice + "stb_image (nothings/stb)\n https://github.com/nothings/stb\n" + + "x (third_party/x; MIT)\n") + out5 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], + capture_output=True, text=True) + chk(out5.returncode == 0, "and naming its path makes the tree rc 0") + (pathlib.Path(tmp) / ".gitmodules").unlink() (pathlib.Path(tmp) / "CMakeLists.txt").write_text("project(x)\n") out3 = subprocess.run([sys.executable, __file__, "--check", "--root", tmp], capture_output=True, text=True) From f96ba3561486092a8b213765f3d4b8b6a7fae26f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 04:20:43 -0400 Subject: [PATCH 3/3] notice_covers_deps: pass --root, because ctest runs these rows in the build directory Review of #2344, and the reviewer's diagnosis is exact: add_python_test() sets no WORKING_DIRECTORY, so ctest runs the row in the BUILD directory, and the check's `--root .` default then looks for CMakeLists.txt where there is none. The self-test passing beside a failing check is that in one line. cd build1336 notice_covers_deps.py --check -> rc 2 'no CMakeLists.txt under .' notice_covers_deps.py --check --root -> rc 0 '3 of 3 ... and 7 of 7 ...' The self-test row needs no root (it builds its own trees) and passes from the build directory too. --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01fe07749..10a5d5116 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -403,7 +403,10 @@ add_python_test(conflict_triage_self ${CMAKE_SOURCE_DIR}/tools/ci/conflict_triag # shipped without a licence line until a reviewer noticed. Matched on the GIT_REPOSITORY URL: that is # what the two files share (`stb` in CMake is `stb_image (nothings/stb)` in NOTICE, so the name is not). add_python_test(notice_covers_deps_self ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_deps.py --self-test) -add_python_test(notice_covers_deps ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_deps.py --check) +# `--root` is explicit: add_python_test() sets no WORKING_DIRECTORY, so ctest runs this row in the BUILD +# directory and a `--root .` default would look for CMakeLists.txt where there is none (review of #2344 -- +# the self-test passed next to a failing check, which is that diagnosis in one line). +add_python_test(notice_covers_deps ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_deps.py --check --root ${CMAKE_SOURCE_DIR}) # reviews: a PR that carries another PR's files lands two merge decisions as one, and one whose work base # already has reads as a fresh duplicate. Both are invisible to a diffstat; this compares blobs. Device-free. add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check.py --self-test)