0.19.2: musl's empty archives, so -lm is answered by the C library and never by the host

.github/workflows/ci.yml

@@ -603,6 +603,73 @@ jobs:
           [ "$bad" = 0 ] || exit 1
           echo "  ok  the internal overlay stops here; the public headers do not"
 
+      # mcpp-community/mcpp#696: A LINK OVER A GRAPH-SUPPLIED C LIBRARY MUST
+      # NOT LET THE HOST ANSWER A NAME THE GRAPH DID NOT.
+      #
+      # `examples/link-stubs` links `-lm`, `-lpthread`, `-ldl` and `-lrt`
+      # explicitly and calls one function through each, and every one of the
+      # four is already defined by musl's own objects, which are on this
+      # program's link line without any of the four. So a correct RUN of the
+      # program is not the evidence a leak was closed: a leak to the host
+      # answers `-lm` silently and the program still builds, runs and reports
+      # correctly, which is exactly why #696 went unnoticed on x86_64 until
+      # something called a symbol only the host's archive had. The evidence
+      # this step reads is the linker's own account of which file answered
+      # each of the four names, and `mcpp build` shows that only with
+      # `--verbose`: a command that succeeds is otherwise reported by mcpp's
+      # own summary line, not by what the command itself printed.
+      #
+      # THIS HOLDS BEFORE AND AFTER mcpp#696's OWN ENGINE FIX. Today, with no
+      # `--sysroot` yet on a graph link, this package's `-L` (mcpp.toml,
+      # beside `port/lib`) precedes the host directories clang and gcc both
+      # still add on this row, so it is what answers all four names first;
+      # once that fix removes those directories from a graph link entirely,
+      # this package's `-L` is the only one left. Either way, the file that
+      # answers each of the four names is this package's own archive, never a
+      # path under `/usr` or `/lib`.
+      - name: The C library answers its own link names, not the host's
+        if: runner.os == 'Linux' && matrix.target == ''
+        working-directory: examples/link-stubs
+        run: |
+          set -euo pipefail
+          pkgroot="$(cd ../.. && pwd)"
+          mcpp build --toolchain '${{ matrix.toolchain }}' --verbose > build.log 2>&1 \
+            || { echo "::error::the build itself failed"; cat build.log; exit 1; }
+
+          bad=0
+          for name in m pthread dl rt; do
+            hits=$(grep -c -- "lib${name}\.a" build.log || true)
+            if [ "$hits" = 0 ]; then
+              echo "::error::-l$name left no trace in the linker's own output — nothing was checked"
+              bad=1
+              continue
+            fi
+            # EVERY line naming this archive, not only the last, so a linker
+            # that logs a failed attempt before a successful one (GNU ld's
+            # own `--verbose` does exactly this) cannot hide a host attempt
+            # behind the eventual, correct answer.
+            others=$(grep -- "lib${name}\.a" build.log \
+                       | grep -v -F -- "$pkgroot/port/lib/lib${name}.a" || true)
+            if [ -n "$others" ]; then
+              echo "::error::-l$name was answered by something other than this package's own archive:"
+              echo "$others" | sed 's/^/        /'
+              bad=1
+            fi
+          done
+          if [ "$bad" != 0 ]; then
+            echo "the full linker trace:"; cat build.log; exit 1
+          fi
+          echo "  ok  -lm -lpthread -ldl -lrt each resolved to $pkgroot/port/lib," \
+               "and nowhere under /usr or /lib"
+
+      # THE SAME FOUR NAMES, RUN RATHER THAN READ. `-lm`, `-lpthread`, `-ldl`
+      # and `-lrt` resolving to an empty archive is only half of "honest": an
+      # empty archive that broke the four calls examples/link-stubs makes
+      # would be answering these names dishonestly in the other direction.
+      - name: The four archives are empty and still correct
+        if: runner.os == 'Linux'
+        run: bash tools/run-probe.sh examples/link-stubs link-stubs
+
       # A program above this package names one package. It does not name
       # openkal, it does not name an implementation, and it says nothing about
       # the platform.

.gitignore

@@ -30,6 +30,7 @@ crash.log
 # What the object-level checks in continuous integration write beside the
 # sources they examine.
 syms.txt
+build.log
 
 # A working tree of the specification or of an implementation placed beside the
 # sources. No trailing slash: the pattern must match a symbolic link as well.

examples/link-stubs/mcpp.toml

@@ -0,0 +1,26 @@
+# mcpp-community/mcpp#696, this package's half (see `port/lib` in the
+# manifest one level up).
+#
+# This program exists to be linked, not to exercise anything novel at run
+# time: `-lm`, `-lpthread`, `-ldl` and `-lrt` are the four of the eight names
+# in `port/lib` that a C program is conventionally written to name explicitly,
+# and this is that program. What `.github/workflows/ci.yml` checks about it is
+# not anything it prints --- it is the linker's own account, read from
+# `mcpp build --verbose`, of which file answered each of those four names. A
+# leak to the host answers silently and this program would still build, run
+# and report correctly on x86_64, which is exactly why #696 went unnoticed
+# there; only the linker's own trace tells the two apart.
+[package]
+name    = "link-stubs"
+version = "0.1.0"
+
+[dependencies]
+openkal-musl = { path = "../.." }
+
+[targets.link-stubs]
+kind = "bin"
+main = "src/main.c"
+
+[build]
+cxx_runtime = "host-coupled"
+ldflags = ["-lm", "-lpthread", "-ldl", "-lrt", "-Wl,--verbose"]

examples/link-stubs/src/main.c

@@ -0,0 +1,41 @@
+/* One call from each of the four archives this fixture links explicitly:
+ * `-lm', `-lpthread', `-ldl' and `-lrt'. Every one of the four functions
+ * below is defined by musl's own objects, already on this program's link
+ * line without any of the four --- so a correct run here is not evidence
+ * that the flags were answered correctly, only that they were not answered
+ * WRONGLY in a way that breaks the program. The evidence this fixture exists
+ * to produce is read from the linker's own trace of the build that produced
+ * this binary (see .github/workflows/ci.yml), not from anything below.
+ */
+#include <stdio.h>
+#include <math.h>
+#include <pthread.h>
+#include <dlfcn.h>
+#include <time.h>
+
+static int failures = 0;
+static void check(int ok, const char *what) {
+	if (!ok) { printf("FAIL: %s\n", what); failures++; }
+	else printf("ok: %s\n", what);
+}
+
+static void *thread_fn(void *arg) { return arg; }
+
+int main(void) {
+	check(fmax(1.0, 2.0) == 2.0, "fmax");
+
+	pthread_t t;
+	void *result = NULL;
+	check(pthread_create(&t, NULL, thread_fn, (void *)1) == 0, "pthread_create");
+	check(pthread_join(t, &result) == 0 && result == (void *)1, "pthread_join");
+
+	struct timespec ts;
+	check(clock_gettime(CLOCK_MONOTONIC, &ts) == 0, "clock_gettime");
+
+	/* A statically linked program cannot dlopen anything; musl reports
+	 * that rather than mishandling it, and this checks the report. */
+	check(dlopen(NULL, RTLD_NOW) == NULL, "dlopen declines on a static binary");
+
+	printf("-- failures: %d --\n", failures);
+	return failures ? 1 : 0;
+}

mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-musl"
-version     = "0.19.1"
+version     = "0.19.2"
 description = "musl 1.2.5 redirected onto openkal: one C library, ported once, above every implementation of the specification rather than above one kernel."
 license     = "Apache-2.0"
 
@@ -488,10 +488,87 @@ cxx_runtime = "host-coupled"
 # arithmetic musl's own <complex.h> calls.
 [target.'cfg(os = "linux")'.build]
 ldflags = ["-nostdlib", "-static", "-Wl,--no-dynamic-linker",
-           "-Wl,--gc-sections"]
+           "-Wl,--gc-sections", "-Lport/lib"]
 # `-lgcc` was here; it is now named by build.mcpp, and only when the compiler
 # is the one whose runtime has that name. See that file.
 
+# `port/lib` HOLDS THE EIGHT ARCHIVES musl'S OWN `make install` PLACES BESIDE
+# `libc.a`, AND THIS PACKAGE NEVER SHIPPED THEM.
+#
+# musl's Makefile states `EMPTY_LIB_NAMES = m rt pthread crypt util xnet
+# resolv dl` and builds each as `ar rc lib<name>.a` with no member named ---
+# eight bytes, `!<arch>\n`, the ar format's magic string and nothing else,
+# the same eight bytes on every host and every architecture. They exist
+# because libc.a already holds every definition POSIX ascribes to `libm`,
+# `librt`, `libpthread`, `libcrypt`, `libutil`, `libxnet`, `libresolv` and
+# `libdl`: musl is one library that answers eight names, not eight
+# libraries, and a program's `-lm` or `-lpthread` is answered by handing the
+# linker a real, empty archive rather than by there being nothing at that
+# name at all. `port/lib` holds the same eight files, built the same way,
+# because this package never ran musl's `make install` and so never
+# produced them.
+#
+# THE ABSENCE WAS INVISIBLE FOR AS LONG AS THE LINK WAS ALLOWED TO LOOK
+# ELSEWHERE. Before mcpp#696, a link whose C library comes from this package
+# --- a graph dependency, not the host's --- still let clang search the
+# build machine's own library directories: `-nostdlib` above removes the
+# startup files and the default libraries, not the search path. So a
+# program's `-lm` was answered by the HOST's `libm`: silently, by a glibc
+# archive, on x86_64-linux-musl (a glibc `s_fmaximum.o` linked into a musl
+# static image, with no diagnostic); loudly, by a linker script naming the
+# wrong object format (`OUTPUT_FORMAT(elf64-x86-64)`), on
+# aarch64-linux-musl, or by `unable to find library -lm` where the host has
+# no such script. mcpp's own repair (mcpp#696) hands a graph link
+# `--sysroot=<an empty directory>`, which removes every host search
+# directory --- and removes the accidental answer along with it, so `-lm`
+# becomes `unable to find library -lm` on every target unless the graph
+# answers it. This package is the graph's C library, so it must.
+#
+# WHY AN EMPTY ARCHIVE, RATHER THAN TELLING A CONSUMER TO DROP THE FLAG.
+# `-lm` on a POSIX system is not a mistake to edit away --- every portable
+# build recipe in existence writes it, and it must keep meaning what it has
+# always meant: a request the linker satisfies by opening a real file. An
+# empty archive lets it do exactly that. The linker opens
+# `port/lib/libm.a`, finds no member the link needs (because nothing outside
+# libc.a defines one), and moves on --- which is the truthful outcome,
+# rather than a special case in this package or in a consumer's manifest
+# that makes the flag disappear before the linker ever sees it.
+#
+# WHY THIS NEEDS NO ENGINE CHANGE. mcpp already resolves a relative `-L` in
+# a dependency's own `[build] ldflags` against that dependency's root
+# directory and forwards the resolved, absolute path to the consumer's link
+# line (the `normalizeDepLdflag` / `propagateLinkFlags` lambdas,
+# `src/build/prepare.cppm`). So `-Lport/lib` above reaches a consumer as
+# `-L<this package's checkout>/port/lib`, without mcpp needing to know
+# anything new about this package.
+#
+# Where that lands relative to a consumer's own `-lm` does not matter.
+# Measured (`clang -###`): the driver collects every `-L` --- a dependency's,
+# in the order its `ldflags` were seen, then whatever host directories it
+# detects because no `--sysroot` is passed yet --- ahead of every `-l` name
+# in the actual invocation it hands to the linker, regardless of where `-l`
+# and `-L` tokens sit in this package's or a consumer's own `ldflags`. This
+# entry is the only explicit `-L` a graph link has today, so it lands first
+# in that list and is consulted, and found, before any host directory: verified
+# against both `ld.lld` (its own `--verbose` names the exact file it opened)
+# and GNU `ld` (`--verbose` reports "attempt to open … succeeded" against
+# this path, and no other) for `-lm`, `-lpthread`, `-ldl` and `-lrt` --- the
+# four names `examples/link-stubs` links and this repository's own CI
+# checks; the search this package's `-L` wins is the same one for the other
+# four, which nothing here links explicitly. Once mcpp#696's sysroot lands,
+# no host directory is searched at all, and this is the only `-L` a graph
+# link has left for these eight names. Either way, nothing else on the
+# command line contends with it.
+#
+# WHY ONLY THIS TARGET, FOR NOW. musl's own interface makes libc the answer
+# to all eight names on every target this package supports, and nothing
+# about that is specific to Linux. This `-L` is added here because this is
+# the target mcpp#696 measured and the one this repository's own CI asserts
+# against (see .github/workflows/ci.yml). Extending it to `cfg(windows)`
+# and `cfg(os = "macos")` is very likely correct and is left for whoever
+# next measures a graph link there, rather than asserted from the reasoning
+# alone.
+
 # The link line on this system.
 #
 # -nostdlib is the statement that the program carries one C library and that it

port/lib/libcrypt.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libdl.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libm.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libpthread.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libresolv.a

@@ -0,0 +1 @@
+!<arch>

port/lib/librt.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libutil.a

@@ -0,0 +1 @@
+!<arch>

port/lib/libxnet.a

@@ -0,0 +1 @@
+!<arch>