diff --git a/CMakeLists.txt b/CMakeLists.txt index 1124dbbb1..caf7f64b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -941,6 +941,12 @@ if(Python3_Interpreter_FOUND) COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/npu_decl_check.py --check --root ${CMAKE_SOURCE_DIR}) add_test(NAME npu_decl_check_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/npu_decl_check.py --self-test) + # engine#2057: no fopen(..., "w"/"a") in the tree without a stated reason -- CodeQL's world-writable class, kept fixed + # by a check rather than by the next scan. Source text only: no build, no device. + add_test(NAME file_create_check + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/file_create_check.py --check --root ${CMAKE_SOURCE_DIR}) + add_test(NAME file_create_check_self + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/file_create_check.py --self-test) # the new-model watcher's alert contract (offline: synthetic batches against the # committed registry, plus FastFlowLM's org listing and releases from the recorded # payloads under tests/fixtures/fflm/ — engine #588), so a regression cannot leave the @@ -1340,6 +1346,15 @@ target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOU # NPU_INFER_DUMP_OPS (npu/forward/include/ops-dump.h, engine#1503): the on/off rule, the mean|x|/absmax arithmetic and # the line's text. Header-only, no device. `add_dependencies(onebit ...)` is what gets it built in CI, whose target # list is explicit (tools/ci/ci_targets_check.py). +# npu/forward/include/file_create.h (engine#2057): the openers every dump and the head-image cache write through -- 0600 / +# at most 0644 whatever the umask, an existing 0666 file narrowed, a symlink refused with ELOOP. Plain C, no model, no +# device; the call sites only swapped fopen() for an opener, so without this arm a revert to fopen() passes every test. +if(NOT WIN32) + add_executable(file_create_test tests/file_create_test.c) + target_include_directories(file_create_test PRIVATE npu/forward/include) + add_test(NAME file_create COMMAND file_create_test) + add_dependencies(onebit file_create_test) +endif() add_executable(ops_dump_test tests/ops_dump_test.cpp) target_include_directories(ops_dump_test PRIVATE npu/forward/include) add_test(NAME ops_dump COMMAND ops_dump_test) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 27472a966..8f8ca8cc9 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -35,6 +35,7 @@ // resets it from position 0. #include "forward_serve.h" +#include "../npu/forward/include/file_create.h" // dumps are created 0600, never through a symlink #include "npu_decode_note.h" #include "../npu/forward/include/q4nx_forward.h" @@ -536,7 +537,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (!e.dump_logits.empty()) { const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + std::to_string(n) + ".bin"; - if (std::FILE* f = std::fopen(path.c_str(), "wb")) { + if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) std::fwrite(logits.data(), sizeof(float), logits.size(), f); std::fclose(f); } diff --git a/docs/npu.md b/docs/npu.md index a54b87124..2040346fd 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2544,6 +2544,13 @@ can fail a run. variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than looking for a code path that skips the dump. +The files these dumps write (`NPU_INFER_DUMP_DIR`'s `layer_.f32`, `NPU_I8_GS_DUMP`'s weight-scale pairs) are +created owner-only (0600) and never through a symlink at the final path component, whatever the umask, and a dump file +an older build left at 0666 is narrowed to 0600 when it is rewritten; the head-image cache's files are at most 0644. Both go through `npu/forward/include/file_create.h`, so no write site uses `fopen(…, "w")`, +which creates with 0666 under the umask and follows a planted link (CodeQL `cpp/world-writable-file-creation`). +`tools/ci/file_create_check.py` refuses a new `fopen(…, "w")` or `"a"` anywhere in the tree unless the line says why +(`file-create-ok: `), and `tests/file_create_test.c` pins the modes. + ``` [op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935 [op] L00 ffn_gate_up rows=1 |x| mean 0.0258482 absmax 1.25728 diff --git a/npu/forward/include/file_create.h b/npu/forward/include/file_create.h new file mode 100644 index 000000000..04756d672 --- /dev/null +++ b/npu/forward/include/file_create.h @@ -0,0 +1,97 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// file_create.h -- create a file for writing with a stated mode, never world-writable, never through a symlink. +// +// fopen(path, "w") creates with 0666 & ~umask: world-writable under a permissive umask, and it follows a symlink that +// someone else planted at a predictable name. CodeQL reports every such site (cpp/world-writable-file-creation), and a +// PR that only touches the surrounding code inherits the alert as "new in code changed by this pull request" -- which +// is how #1830 sat red for a scan it did not cause. These two openers are what the env-gated dumps and the head-image +// cache use instead. The mode is a constant inside each function, so the scan can see it. +// onebit_fopen_private_wb 0600: dumps and probes (an activation or weight dump is nobody else's business) +// onebit_fopen_shared_wb 0644: the head-image cache, which a later serve reads back. A cache file only; anything +// that holds activations, weights or a request is a dump and takes the 0600 one. +// onebit_fopen_private_ab 0600, appending: a dump log several runs add to (ort_run's outputs.txt) +// The two `wb` openers truncate an existing file (the `ab` one appends); all three refuse a symlink at the final component (O_NOFOLLOW), and leave errno from the +// failing call, so a caller's strerror(errno) still names the open's reason. O_CREAT's mode applies only when the file +// is created, so a regular file that already exists -- one an older build wrote 0666 -- has every bit beyond the stated +// mode cleared with fchmod() as well. That only narrows: a stricter umask still wins (a 077 umask leaves the cache 0600), +// and a device or pseudo-file (/dev/full, /proc/...) keeps its own mode. Pinned by tests/file_create_test.c, +// and tools/ci/file_create_check.py refuses a new fopen(..., "w"/"a") anywhere in the tree. +// Plain C, so .c and .cpp sources share it. +#pragma once + +#include +#include +#if !defined(_WIN32) +#include +#include +#include +#endif + +#if !defined(_WIN32) +static inline FILE* onebit_fdopen_(int fd, mode_t mode, const char* how) { + if (fd < 0) return NULL; + struct stat st; + if (fstat(fd, &st) != 0) { + const int e = errno; + close(fd); + errno = e; + return NULL; + } + // Only ever narrows: bits beyond the stated mode are cleared, and none is added, so a stricter umask still wins. + if (S_ISREG(st.st_mode) && ((st.st_mode & 07777) & ~mode) != 0 && fchmod(fd, (st.st_mode & 07777) & mode) != 0) { + const int e = errno; + close(fd); + errno = e; + return NULL; + } + FILE* f = fdopen(fd, how); + if (!f) { + const int e = errno; + close(fd); + errno = e; + } + return f; +} +#endif + +static inline FILE* onebit_fopen_private_wb(const char* path) { +#if defined(_WIN32) + return fopen(path, "wb"); // file-create-ok: Windows has no POSIX mode; the CodeQL finding is POSIX-only +#else + return onebit_fdopen_(open(path, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, S_IRUSR | S_IWUSR), + S_IRUSR | S_IWUSR, "wb"); +#endif +} + +static inline FILE* onebit_fopen_shared_wb(const char* path) { +#if defined(_WIN32) + return fopen(path, "wb"); // file-create-ok: Windows has no POSIX mode; the CodeQL finding is POSIX-only +#else + return onebit_fdopen_( + open(path, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH), + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH, "wb"); +#endif +} + +static inline FILE* onebit_fopen_private_ab(const char* path) { +#if defined(_WIN32) + return fopen(path, "ab"); // file-create-ok: Windows has no POSIX mode; the CodeQL finding is POSIX-only +#else + return onebit_fdopen_(open(path, O_WRONLY | O_CREAT | O_APPEND | O_NOFOLLOW | O_CLOEXEC, S_IRUSR | S_IWUSR), + S_IRUSR | S_IWUSR, "ab"); +#endif +} diff --git a/npu/forward/include/residual-dump.h b/npu/forward/include/residual-dump.h index 3d6cb41df..b4e00e426 100644 --- a/npu/forward/include/residual-dump.h +++ b/npu/forward/include/residual-dump.h @@ -31,12 +31,14 @@ #include #include +#include "file_create.h" // the dump is created 0600, never through a symlink + namespace onebit::npu { // Writes `count` floats to `path`. Returns false and fills `err` (path + strerror) when the file cannot be created, the // write is short, or the close fails. `err` is left untouched on success. inline bool write_residual_dump(const char * path, const float * data, size_t count, std::string * err) { - FILE * fp = std::fopen(path, "wb"); + FILE * fp = onebit_fopen_private_wb(path); if (fp == nullptr) { if (err != nullptr) { *err = std::string(path) + ": " + std::strerror(errno); diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e28008415..6eaaf186e 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -27,6 +27,7 @@ #include "batch_width.h" #include "ffn_cache_policy.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested +#include "file_create.h" // dumps are created 0600, never through a symlink #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) #include #include @@ -3284,7 +3285,7 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, static int n = 0; char path[4096]; std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); - if (std::FILE* f = std::fopen(path, "wb")) { + if (std::FILE* f = onebit_fopen_private_wb(path)) { std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); std::fclose(f); } diff --git a/npu/forward/src/q4nx_head_image.c b/npu/forward/src/q4nx_head_image.c index 33751e029..109fb6b64 100644 --- a/npu/forward/src/q4nx_head_image.c +++ b/npu/forward/src/q4nx_head_image.c @@ -18,6 +18,7 @@ // geometry and a hash, and a reader that cannot reproduce the hash refuses instead of returning bytes it cannot vouch for. #include "q4nx_head_image.h" +#include "file_create.h" /* the cache file is created 0644, never through a symlink */ #include #include @@ -82,7 +83,7 @@ int q4nx_head_image_write(const char *path, const int8_t *codes, const float *sc const int tn = snprintf(tmp, sizeof tmp, "%s.tmp", path); #endif if (tn <= 0 || (size_t)tn >= sizeof tmp) return 0; - FILE *f = fopen(tmp, "wb"); + FILE *f = onebit_fopen_shared_wb(tmp); if (!f) return 0; int ok = fwrite(&h, sizeof h, 1, f) == 1 && fwrite(codes, (size_t)kd * (size_t)nd, 1, f) == 1 && fwrite(scales, (size_t)nd * sizeof(float), 1, f) == 1; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 43e6ef982..fe1a67fcb 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -21,6 +21,7 @@ // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" #include "batch_width.h" +#include "file_create.h" // dumps are created 0600, never through a symlink #include #include @@ -1262,7 +1263,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: static int n = 0; char path[4096]; std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); - if (std::FILE* f = std::fopen(path, "wb")) { + if (std::FILE* f = onebit_fopen_private_wb(path)) { if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); std::fclose(f); } diff --git a/onnx/ep/onebit_ep_qdq_proof.cpp b/onnx/ep/onebit_ep_qdq_proof.cpp index 2d4ca7889..55009c9bd 100644 --- a/onnx/ep/onebit_ep_qdq_proof.cpp +++ b/onnx/ep/onebit_ep_qdq_proof.cpp @@ -547,7 +547,7 @@ long proc_kb(const char* key) { // Writing 5 to clear_refs resets VmHWM, so the next run's peak is the run's own. bool reset_peak() { - FILE* f = std::fopen("/proc/self/clear_refs", "w"); + FILE* f = std::fopen("/proc/self/clear_refs", "w"); // file-create-ok: a /proc pseudo-file, never created if (!f) return false; const bool ok = std::fputc('5', f) != EOF; std::fclose(f); diff --git a/onnx/ort_run.cpp b/onnx/ort_run.cpp index 7099e84c7..ccafc08b5 100644 --- a/onnx/ort_run.cpp +++ b/onnx/ort_run.cpp @@ -38,6 +38,8 @@ #include +#include "../npu/forward/include/file_create.h" // dumps are created 0600, never through a symlink + #include #include #include @@ -722,7 +724,7 @@ int run_generic(OrtSession* session, OrtMemoryInfo* mi, const Args& a) { } if (a.dump_all) ::mkdir(a.dump_all, 0700); // EEXIST is the normal case - FILE* side = a.dump_all ? std::fopen((std::string(a.dump_all) + "/outputs.txt").c_str(), "a") : nullptr; + FILE* side = a.dump_all ? onebit_fopen_private_ab((std::string(a.dump_all) + "/outputs.txt").c_str()) : nullptr; for (size_t i = 0; i < outs.size(); ++i) { OrtAllocator* alloc = nullptr; char* gname = nullptr; @@ -758,7 +760,7 @@ int run_generic(OrtSession* session, OrtMemoryInfo* mi, const Args& a) { size_t elems = 1; for (int64_t d : dims) elems *= (size_t)(d > 0 ? d : 1); const size_t bytes = elems * elem_size(et); - FILE* f = std::fopen(path.c_str(), "wb"); + FILE* f = onebit_fopen_private_wb(path.c_str()); if (!f) { std::fprintf(stderr, "onebit-ort-run: cannot write %s\n", path.c_str()); rc = 2; diff --git a/tests/file_create_test.c b/tests/file_create_test.c new file mode 100644 index 000000000..4836ab1b9 --- /dev/null +++ b/tests/file_create_test.c @@ -0,0 +1,143 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// file_create_test.c -- npu/forward/include/file_create.h: the modes the two openers promise, and the symlink refusal. +// +// The fix this pins is invisible without an arm: every call site only swapped fopen() for an opener, and a later +// "simplification" back to fopen() would pass every other test. So, with umask 0 (the case where fopen gives 0666): +// [1] a new file from the private opener is 0600, and from the shared opener 0644; +// [2] a file that already exists wider than the stated mode -- 0664, what fopen() left under a 002 umask; the test +// does not make a world-writable file itself, which CodeQL would rightly flag -- is narrowed to 0600 / 0644 when +// rewritten, and its old contents are truncated; +// [3] a stricter umask still wins: under 077 the shared opener's new file is 0600, not widened to 0644; +// [4] a symlink at the path is refused with ELOOP, and the file it points at is untouched; +// [5] a failure leaves the open's errno (a missing directory reads ENOENT), so a caller's strerror still says why; +// [6] a device keeps its own mode: /dev/null opens, and is not chmod-ed; +// [7] the append opener creates 0600, keeps what is there, and narrows an existing 0664 log. +// Scratch goes under ONEBIT_TEST_TMP, else ~/.cache/scratch-tests/tmp (tests/tmp_root.h's rule), and is removed. +#include "file_create.h" + +#include +#include +#include +#include +#include +#include + +static int fails = 0; +static void check(int ok, const char* what) { + printf("%s %s\n", ok ? "PASS" : "FAIL", what); + if (!ok) fails++; +} + +static int mode_of(const char* p) { + struct stat st; + return stat(p, &st) == 0 ? (int)(st.st_mode & 07777) : -1; +} + +static long size_of(const char* p) { + struct stat st; + return stat(p, &st) == 0 ? (long)st.st_size : -1; +} + +static int write_one(FILE* f, const char* s) { + if (!f) return 0; + const int ok = fputs(s, f) >= 0; + return fclose(f) == 0 && ok; +} + +int main(void) { + char root[1024]; + const char* e = getenv("ONEBIT_TEST_TMP"); + const char* h = getenv("HOME"); + if (e && *e) snprintf(root, sizeof root, "%s/file_create_test.XXXXXX", e); + else if (h && *h) { + char base[900]; + snprintf(base, sizeof base, "%s/.cache/scratch-tests/tmp", h); + char mk[1000]; + snprintf(mk, sizeof mk, "%s/.cache", h); mkdir(mk, 0700); + snprintf(mk, sizeof mk, "%s/.cache/scratch-tests", h); mkdir(mk, 0700); + mkdir(base, 0700); + snprintf(root, sizeof root, "%s/file_create_test.XXXXXX", base); + } else snprintf(root, sizeof root, "/tmp/file_create_test.XXXXXX"); + if (!mkdtemp(root)) { + printf("FAIL mkdtemp(%s): %s\n", root, strerror(errno)); + return 1; + } + char a[1200], b[1200], c[1200], l[1200], t[1200], n[1200]; + snprintf(a, sizeof a, "%s/private", root); + snprintf(b, sizeof b, "%s/shared", root); + snprintf(c, sizeof c, "%s/strict", root); + snprintf(l, sizeof l, "%s/link", root); + snprintf(t, sizeof t, "%s/target", root); + snprintf(n, sizeof n, "%s/no-such-dir/x", root); + + umask(0); + check(write_one(onebit_fopen_private_wb(a), "x") && mode_of(a) == 0600, "[1] a new private file is 0600 under umask 0"); + check(write_one(onebit_fopen_shared_wb(b), "x") && mode_of(b) == 0644, "[1] a new shared file is 0644 under umask 0"); + + chmod(a, 0664); + chmod(b, 0664); + check(mode_of(a) == 0664 && mode_of(b) == 0664, "[2] the two files start wider than their openers allow (0664)"); + check(write_one(onebit_fopen_private_wb(a), "") && mode_of(a) == 0600 && size_of(a) == 0, + "[2] an existing 0664 file is narrowed to 0600 and truncated"); + check(write_one(onebit_fopen_shared_wb(b), "") && mode_of(b) == 0644, "[2] an existing 0664 cache file is narrowed to 0644"); + + umask(077); + check(write_one(onebit_fopen_shared_wb(c), "x") && mode_of(c) == 0600, + "[3] under umask 077 the shared opener's new file is 0600 (narrow-only, never widened)"); + umask(0); + + { + FILE* f = onebit_fopen_private_wb(t); // the symlink's target, created 0600 and expected to stay untouched + if (f) { fputs("keep", f); fclose(f); } + } + check(symlink(t, l) == 0, "a symlink to a target file"); + errno = 0; + FILE* f = onebit_fopen_private_wb(l); + const int e1 = errno; + check(f == NULL && e1 == ELOOP, "[4] the private opener refuses a symlink with ELOOP"); + if (f) fclose(f); + errno = 0; + f = onebit_fopen_shared_wb(l); + const int e2 = errno; + check(f == NULL && e2 == ELOOP, "[4] the shared opener refuses a symlink with ELOOP"); + if (f) fclose(f); + check(size_of(t) == 4 && mode_of(t) == 0600, "[4] the symlink's target is untouched (size and mode)"); + + errno = 0; + f = onebit_fopen_private_wb(n); + check(f == NULL && errno == ENOENT, "[5] a missing directory fails with ENOENT, for the caller's strerror"); + + struct stat before, after; + const int had = stat("/dev/null", &before) == 0; + f = onebit_fopen_private_wb("/dev/null"); + check(f != NULL, "[6] a device opens"); + if (f) fclose(f); + check(!had || (stat("/dev/null", &after) == 0 && (after.st_mode & 07777) == (before.st_mode & 07777)), + "[6] and keeps its own mode (it is not chmod-ed to 0600)"); + + char g[1200]; + snprintf(g, sizeof g, "%s/log", root); + check(write_one(onebit_fopen_private_ab(g), "ab") && mode_of(g) == 0600, "[7] the append opener creates 0600"); + chmod(g, 0664); + check(write_one(onebit_fopen_private_ab(g), "cd") && size_of(g) == 4 && mode_of(g) == 0600, + "[7] a second append keeps the first line and narrows a 0664 log to 0600"); + + unlink(a); unlink(b); unlink(c); unlink(l); unlink(t); unlink(g); + rmdir(root); + printf("%s (%d failed)\n", fails ? "FAILED" : "ALL PASS", fails); + return fails ? 1 : 0; +} diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index dfceaa69a..700dd6bfb 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -33,6 +33,7 @@ // their biases, the MLP projections and the final norm -- and produces a container the engine's own // Model can open. +#include "../npu/forward/include/file_create.h" // written files are 0600, never through a symlink #include "model.h" #include "pack.h" #include "pack_import.h" @@ -4617,7 +4618,7 @@ static void dump_real_component(const std::string& dir) { const int row = 1; // token id 1: a plain text token ORT embeds if ((row + 1) * H > v.size()) return; const std::string out = dir + "/eng_row.f32"; - if (FILE* f = std::fopen(out.c_str(), "wb")) { + if (FILE* f = onebit_fopen_private_wb(out.c_str())) { std::fwrite(&v[size_t(row) * H], sizeof(float), H, f); std::fclose(f); double s = 0; diff --git a/tests/npu_residual_dump_test.cpp b/tests/npu_residual_dump_test.cpp index f64c97579..353f0be66 100644 --- a/tests/npu_residual_dump_test.cpp +++ b/tests/npu_residual_dump_test.cpp @@ -67,7 +67,7 @@ int main() { // 3. a short write: /dev/full accepts the open and fails the write, which is the full-disk case, and it is the arm a // version that only checks fopen would pass. err.clear(); - if (FILE * probe = std::fopen("/dev/full", "wb")) { // skips cleanly on a host without it + if (FILE * probe = std::fopen("/dev/full", "rb")) { // skips cleanly on a host without it (read: a probe creates nothing) std::fclose(probe); check(!onebit::npu::write_residual_dump("/dev/full", data.data(), data.size(), &err), "a full disk is a failure even though fopen succeeded"); diff --git a/tools/ci/file_create_check.py b/tools/ci/file_create_check.py new file mode 100755 index 000000000..1d973bf16 --- /dev/null +++ b/tools/ci/file_create_check.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Refuse a new fopen(..., "w"/"a") in the tree's C and C++ sources (CodeQL cpp/world-writable-file-creation). + +fopen() creates with 0666 & ~umask -- world-writable under a permissive umask -- and follows a symlink planted at the +name. CodeQL reports each such site, and a PR that only touches nearby code inherits the alert as new: #1830 sat on a +CHANGES verdict for two dump sites it did not write. #2057 moved every write site onto npu/forward/include/file_create.h +(0600 dumps, at most 0644 for the head-image cache, O_NOFOLLOW), and this check keeps it that way: the next dump someone +adds will not know about the header, and CodeQL would otherwise find it a week later. + +A line may keep fopen for writing when it says why, on the same line: `// file-create-ok: ` (a /proc +pseudo-file, the Windows fallback, a test fixture). An empty reason does not count. Matches inside a // comment are +ignored, so prose that quotes the call is fine. + +usage: + tools/ci/file_create_check.py --check [--root DIR] + tools/ci/file_create_check.py --self-test +""" +import argparse +import os +import re +import subprocess +import sys +import tempfile + +PATTERN = re.compile(r'\bfopen\s*\([^;]*?,\s*"(?:w|a)[^"]*"') +MARKER = re.compile(r"file-create-ok:\s*\S") +EXTS = (".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp") + + +def offending(text): + """[(line_no, line)] of the write-mode fopen calls in `text` that carry no reason.""" + out = [] + for no, line in enumerate(text.splitlines(), 1): + m = PATTERN.search(line) + if not m: + continue + comment = line.find("//") + if comment != -1 and comment < m.start(): + continue # prose that quotes the call + if line.lstrip().startswith(("*", "/*")): + continue + if MARKER.search(line): + continue + out.append((no, line.strip())) + return out + + +def tracked_sources(root): + r = subprocess.run(["git", "-C", root, "ls-files", "-z"], capture_output=True, check=True) + return [p for p in r.stdout.decode().split("\0") if p.endswith(EXTS)] + + +def check(root): + bad = [] + for rel in tracked_sources(root): + try: + with open(os.path.join(root, rel), encoding="utf-8", errors="replace") as f: + text = f.read() + except OSError: + continue + bad += [(rel, no, line) for no, line in offending(text)] + for rel, no, line in bad: + print(f"{rel}:{no}: fopen for writing creates 0666 and follows a symlink: {line}") + if bad: + print(f"file_create_check: {len(bad)} site(s). Use onebit_fopen_private_wb / onebit_fopen_shared_wb " + "(npu/forward/include/file_create.h), or say why on the line: // file-create-ok: ") + return 1 + print("file_create_check: no fopen for writing without a stated reason") + return 0 + + +def self_test(): + fails = 0 + + def t(ok, what): + nonlocal fails + print(("ok " if ok else "FAIL ") + what) + fails += 0 if ok else 1 + + t(len(offending('FILE* f = fopen(path, "wb");')) == 1, "a plain write-mode fopen is refused") + t(len(offending('if (std::FILE* f = std::fopen(path, "wb")) {')) == 1, "std::fopen inside an if is refused") + t(len(offending('FILE* f = fopen(p, "a");')) == 1, "append mode is refused (it creates too)") + t(len(offending('FILE* f = fopen(p, "w+");')) == 1, "w+ is refused") + t(offending('FILE* f = fopen(p, "rb");') == [], "a read is fine") + t(offending('FILE* f = fopen(p, "r+");') == [], "r+ does not create, so it is fine") + t(offending('f = fopen("/proc/self/clear_refs", "w"); // file-create-ok: a /proc pseudo-file') == [], + "a stated reason is accepted") + t(len(offending('f = fopen(p, "w"); // file-create-ok:')) == 1, "an empty reason is not") + t(offending('// fopen(path, "w") creates with 0666') == [], "prose in a // comment is ignored") + t(offending(' * fopen(path, "w") in a block comment') == [], "prose in a block comment is ignored") + t(offending('FILE* f = onebit_fopen_private_wb(path);') == [], "the openers themselves are fine") + # End to end on a throwaway repository: one bad file fails the check, the fixed file passes it. + with tempfile.TemporaryDirectory() as d: + def git(*a): + subprocess.run(["git", "-C", d, *a], check=True, capture_output=True) + git("init", "-q") + with open(os.path.join(d, "x.c"), "w") as f: + f.write('void f(const char* p) { FILE* o = fopen(p, "wb"); }\n') + git("add", "x.c") + t(check(d) == 1, "--check fails on a tracked file with the pattern") + with open(os.path.join(d, "x.c"), "w") as f: + f.write('void f(const char* p) { FILE* o = onebit_fopen_private_wb(p); }\n') + t(check(d) == 0, "--check passes once it uses the opener") + print("file_create_check self-test:", "ok" if fails == 0 else f"{fails} failed") + return 0 if fails == 0 else 1 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--check", action="store_true") + ap.add_argument("--root", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args() + if a.self_test: + return self_test() + if a.check: + return check(os.path.abspath(a.root)) + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main())