Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0")
bazel_dep(name = "ncurses", version = "6.4.20221231.bcr.4")

test_tool_sources = use_extension("//internal/test_tools:repositories.bzl", "sources")
use_repo(test_tool_sources, "test_tools_libmagic", "test_tools_procps_ng", "test_tools_toybox")
use_repo(test_tool_sources, "test_tools_libmagic", "test_tools_toybox")
5 changes: 1 addition & 4 deletions internal/remote.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,7 @@ def _native_test(ctx, root, descriptor, files, job):
"unset BASH_ENV",
'if [[ -e /bin/sh || -e /lib64/ld-linux-x86-64.so.2 ]]; then echo "Browser tests require isolated actiond execution" >&2; exit 1; fi',
'case "$TEST_SRCDIR" in /*) ;; *) export TEST_SRCDIR="$PWD/$TEST_SRCDIR" ;; esac',
'export MAGIC="$TEST_SRCDIR/%s"' % runfile(tools.magic),
] + [
'%s() { "$TEST_SRCDIR/%s" %s "$@"; }; export -f %s' % (command, runfile(binary), " ".join(args), command)
for command, (binary, args) in tools.commands.items()
'source "$TEST_SRCDIR/%s"' % runfile(tools.shell_setup),
]) + "\n")
ctx.actions.write(executable, "\n".join([
"#!/bin/bash",
Expand Down
18 changes: 13 additions & 5 deletions internal/test_tools/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@ test_tools(
name = "tools",
file = "@test_tools_libmagic//:file",
magic = "@test_tools_libmagic//:magic.mgc",
pgrep = "@test_tools_procps_ng//:pgrep",
toybox = "@test_tools_toybox//:toybox",
zip = "@zip//:zip",
visibility = ["//:__subpackages__"],
zip = "@zip//:zip",
)

py_binary(
Expand All @@ -23,18 +22,27 @@ py_test(
srcs = ["tools_test.py"],
args = ["$(rlocationpaths :tools)"],
data = [":tools"],
target_compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
deps = ["@rules_python//python/runfiles"],
target_compatible_with = ["@platforms//os:linux", "@platforms//cpu:x86_64"],
)

config_setting(
name = "linux_x86_64",
constraint_values = ["@platforms//os:linux", "@platforms//cpu:x86_64"],
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_aarch64",
constraint_values = ["@platforms//os:linux", "@platforms//cpu:aarch64"],
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:aarch64",
],
visibility = ["//visibility:public"],
)
17 changes: 10 additions & 7 deletions internal/test_tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,31 @@
Bazel's `test-setup.sh` executes before our native browser test. Its utilities
are declared test runfiles, built for Linux x86-64 with hermetic LLVM and musl:

- Toybox: filesystem operations, find, grep, sed, and ps.
- procps-ng: pgrep, including Bazel's `-a -g` process-group query.
- Toybox: filesystem operations, find, grep, sed, ps, and pgrep.
- libmagic: file and its compiled MIME database.
- Info-ZIP: packaging undeclared test outputs.

All four executables are static. The launcher calls their runfile paths directly
All three executables are static. The launcher calls their runfile paths directly
through `BASH_ENV` functions and sets `MAGIC` to the declared database. actiond
still supplies the pinned static Bash requested by `requires-bash`. Browser
libraries and fonts remain separate, caller-selectable runtime inputs.

The private launcher adapter maps Bazel's exact `pgrep -a -g PGID` probe to
Toybox's `pgrep -g PGID`. Bazel only checks for nonempty output; it does not
consume the full command line requested by procps's `-a` flag. Other invocations
retain Toybox's native argument handling.

The tools' platform transition selects the pinned LLVM toolchain only for this
bundle, without replacing the caller's C/C++ toolchains globally. The libmagic
database compiler runs on the build execution platform and also uses musl on
Linux; cross-compiling test tools from macOS does not execute Linux binaries.

`repositories.bzl` pins source archives and BCR overlays. Three small local
`repositories.bzl` pins source archives and BCR overlays. Two small local
patches are applied through repository rules so they also apply in consuming
modules (root-only module overrides would not):

- Toybox: use musl's syslog-name definitions in the compilation unit using them.
- procps-ng: remove unavailable glibc/gettext feature claims and generate the
empty config header with a Bazel write action.
- Toybox: use musl's syslog-name definitions in the compilation unit using them,
and make `pgrep -g` select process groups instead of the Unix groups used by `ps -g`.
- libmagic: compile its database using declared Python/compiler inputs instead
of host cat/mv/rm commands, with a static Linux build-time compiler.

Expand Down
26 changes: 21 additions & 5 deletions internal/test_tools/defs.bzl
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Declared static Linux utilities used by Bazel's native test launcher."""

TestToolsInfo = provider(fields = ["commands", "magic"])
load("//playwright:defs.bzl", "runfile")

TestToolsInfo = provider(fields = ["shell_setup"])

_MULTICALL_COMMANDS = ["cat", "date", "dirname", "find", "grep", "ln", "mkdir", "ps", "rm", "sed", "sleep", "sort", "stat", "touch"]

Expand All @@ -18,15 +20,29 @@ linux_tools = transition(

def _tools_impl(ctx):
commands = {name: (ctx.executable.toybox, [name]) for name in _MULTICALL_COMMANDS}
commands.update({name: (getattr(ctx.executable, name), []) for name in ["file", "pgrep", "zip"]})
files = depset([ctx.executable.toybox, ctx.executable.file, ctx.executable.pgrep, ctx.executable.zip, ctx.file.magic])
return [DefaultInfo(files = files, runfiles = ctx.runfiles(transitive_files = files)), TestToolsInfo(commands = commands, magic = ctx.file.magic)]
commands.update({name: (getattr(ctx.executable, name), []) for name in ["file", "zip"]})
setup = ctx.actions.declare_file(ctx.label.name + ".bash-env")
ctx.actions.write(setup, "\n".join([
'export MAGIC="$TEST_SRCDIR/%s"' % runfile(ctx.file.magic),
] + [
'%s() { "$TEST_SRCDIR/%s" %s "$@"; }; export -f %s' % (command, runfile(binary), " ".join(args), command)
for command, (binary, args) in commands.items()
] + [
# Bazel checks only whether this exact process-group probe has output.
# Toybox has -g but not procps's -a (print full command line).
"pgrep() {",
" if [[ $# == 3 && $1 == -a && $2 == -g ]]; then shift; fi",
' "$TEST_SRCDIR/%s" pgrep "$@"' % runfile(ctx.executable.toybox),
"}; export -f pgrep",
]) + "\n")
files = depset([ctx.executable.toybox, ctx.executable.file, ctx.executable.zip, ctx.file.magic, setup])
return [DefaultInfo(files = files, runfiles = ctx.runfiles(transitive_files = files)), TestToolsInfo(shell_setup = setup)]

test_tools = rule(
implementation = _tools_impl,
cfg = linux_tools,
attrs = dict(
{name: attr.label(executable = True, cfg = "target", mandatory = True) for name in ["toybox", "file", "pgrep", "zip"]},
{name: attr.label(executable = True, cfg = "target", mandatory = True) for name in ["toybox", "file", "zip"]},
_allowlist_function_transition = attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
magic = attr.label(allow_single_file = True, mandatory = True),
),
Expand Down
5 changes: 4 additions & 1 deletion internal/test_tools/patches/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
exports_files(["libmagic.patch", "procps-ng.patch", "toybox.patch"])
exports_files([
"libmagic.patch",
"toybox.patch",
])
76 changes: 0 additions & 76 deletions internal/test_tools/patches/procps-ng.patch

This file was deleted.

38 changes: 38 additions & 0 deletions internal/test_tools/patches/toybox.patch
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
diff --git a/toys/pending/syslogd.c b/toys/pending/syslogd.c
--- a/toys/pending/syslogd.c
+++ b/toys/pending/syslogd.c
@@ -31,6 +31,12 @@
Expand All @@ -13,3 +14,40 @@

#define FOR_syslogd
#include "toys.h"
diff --git a/toys/posix/ps.c b/toys/posix/ps.c
--- a/toys/posix/ps.c
+++ b/toys/posix/ps.c
@@ -463,10 +463,10 @@
}

// process match filter for top/ps/pgrep: Return 0 to discard, nonzero to keep
-static int shared_match_process(long long *slot)
+static int match_process_with_group(long long *slot, int group_slot)
{
struct ps_ptr_len match[] = {
- {&TT.gg, SLOT_gid}, {&TT.GG, SLOT_rgid}, {&TT.pp, SLOT_pid},
+ {&TT.gg, group_slot}, {&TT.GG, SLOT_rgid}, {&TT.pp, SLOT_pid},
{&TT.PP, SLOT_ppid}, {&TT.ss, SLOT_sid}, {&TT.tt, SLOT_ttynr},
{&TT.uu, SLOT_uid}, {&TT.UU, SLOT_ruid}
};
@@ -484,6 +484,11 @@
}

return ll ? 0 : -1;
+}
+
+static int shared_match_process(long long *slot)
+{
+ return match_process_with_group(slot, SLOT_gid);
}

// process match filter for ps: Return 0 to discard, nonzero to keep
@@ -1945,7 +1950,7 @@

static int pgrep_match_process(long long *slot)
{
- return !FLAG(v) == !!shared_match_process(slot);
+ return !FLAG(v) == !!match_process_with_group(slot, SLOT_pgrp);
}

void pgrep_main(void)
14 changes: 0 additions & 14 deletions internal/test_tools/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,6 @@ def _sources_impl(_ctx):
patch_args = ["-p1"],
patches = ["//internal/test_tools/patches:toybox.patch"],
)
http_archive(
name = "test_tools_procps_ng",
urls = ["https://gitlab.com/procps-ng/procps/-/archive/v4.0.5/procps-v4.0.5.tar.gz"],
integrity = "sha256-LG1+2fKs3h1N1GAsYXL+Vu/4aVP+hjm9Yz29IswY9ds=",
strip_prefix = "procps-v4.0.5",
remote_file_urls = {
"BUILD": [_BCR + "/procps-ng/4.0.5/overlay/BUILD"],
},
remote_file_integrity = {
"BUILD": "sha256-g/zOMIvfZisLKcYxUGyco/iglnwU+Iz4vnXkvEKPw3g=",
},
patch_args = ["-p1"],
patches = ["//internal/test_tools/patches:procps-ng.patch"],
)
http_archive(
name = "test_tools_libmagic",
urls = ["https://github.com/file/file/archive/refs/tags/FILE5_47.tar.gz"],
Expand Down
35 changes: 26 additions & 9 deletions internal/test_tools/tools_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import base64
import os
from pathlib import Path
import signal
import struct
import subprocess
import sys
Expand All @@ -25,11 +24,13 @@ def setUp(self):
self.env = dict(os.environ, PATH="", MAGIC=_FILES["magic.mgc"], LC_ALL="C")

def run_tool(self, command, *args):
prefix = [_FILES[command]] if command in ("file", "pgrep", "zip") else [_FILES["toybox"], command]
return subprocess.check_output(prefix + list(args), cwd=self.root, env=self.env, text=True)
return subprocess.check_output(
["/bin/bash", "-c", 'source "$1"; shift; "$@"', "tools", _FILES["tools.bash-env"], command, *args],
cwd=self.root, env=self.env, text=True,
)

def test_static_elf(self):
for name in ("toybox", "file", "pgrep", "zip"):
for name in ("toybox", "file", "zip"):
with self.subTest(name=name):
data = Path(_FILES[name]).read_bytes()
self.assertEqual(data[:6], b"\x7fELF\x02\x01")
Expand Down Expand Up @@ -69,13 +70,29 @@ def test_manifest_and_signal_parsing(self):
self.assertTrue(self.run_tool("date", "+%F %T %Z").strip())

def test_process_group_monitoring(self):
process = subprocess.Popen([_FILES["toybox"], "sleep", "60"], env=self.env, start_new_session=True)
leader = subprocess.Popen([_FILES["toybox"], "sleep", "60"], env=self.env, process_group=0)
member = None
try:
self.assertIn(str(process.pid), self.run_tool("ps", "-p", str(process.pid)))
self.assertIn(str(process.pid), self.run_tool("pgrep", "-a", "-g", str(process.pid)))
member = subprocess.Popen([_FILES["toybox"], "sleep", "60"], env=self.env, process_group=leader.pid)
group = str(leader.pid)
self.assertIn(group, self.run_tool("ps", "-p", group, "-o", "PID=").split())
self.assertIn(group, self.run_tool("ps", "-g", str(os.getgid()), "-o", "PID=").split())
self.assertEqual(set(self.run_tool("pgrep", "-a", "-g", group).split()), {group, str(member.pid)})
leader.terminate()
leader.wait(timeout=5)
# The group is still alive after its leader exits: cleanup must wait.
self.assertEqual(self.run_tool("pgrep", "-a", "-g", group).strip(), str(member.pid))
member.terminate()
member.wait(timeout=5)
with self.assertRaises(subprocess.CalledProcessError) as failure:
self.run_tool("pgrep", "-a", "-g", group)
self.assertEqual(failure.exception.returncode, 1)
self.assertEqual(failure.exception.output, "")
finally:
os.killpg(process.pid, signal.SIGTERM)
process.wait(timeout=5)
for process in (leader, member):
if process is not None and process.poll() is None:
process.terminate()
process.wait(timeout=5)

def test_screenshot_mime_and_zip(self):
png = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jZ1kAAAAASUVORK5CYII=")
Expand Down
Loading