diff --git a/deploy/install/README.md b/deploy/install/README.md index 6d551fd23..9e2f4d392 100644 --- a/deploy/install/README.md +++ b/deploy/install/README.md @@ -33,6 +33,7 @@ This directory holds the Core/Web installer, the `oac` command and the node inst - Installation flags only seed `config.json`. A rerun of the installer accepts only `--install-dir` and repairs. - `oac apply` validates `config.json`, derives `generated/` and converges on what actually runs. Each service carries the digest of its inputs (Compose label `io.oac.inputs`, native `OAC_INPUTS`), and exactly the services whose running inputs differ are recreated or restarted. Decide restarts from what runs, never from recorded bookkeeping, so the next apply finishes an interrupted one. Apply contacts running services at the last applied address before changing listeners. - Health checks, setup, apply and generated service files derive addresses from the same `config.json`. `host` selects the gateway listener for managed ingress and the Core and Web listeners for external ingress. A non-loopback external bind requires an HTTPS public origin. +- `configuration.listeners` lists every host listener from the same helpers that render the port mappings and listen addresses, so port checks and real binds cannot diverge. A new installation checks them all before it hashes the bundle or loads images, and `oac apply` checks those a change adds before it touches a service; the installation's own listeners do not count. The probe binds with `SO_REUSEADDR`, as the services do. Where a port overlaps one of the installation's own listeners, or the account may not bind a port below 1024, it reads the kernel's listening sockets instead. - Runtime settings stay in PostgreSQL and change through Web or `/core/v1`. Secrets live once each in `secrets/`; identity and installation facts live in the tool-written `state.json` (format 2, with an `oac-` Compose project). `config.json` has its own format, 1. - Core reads only its environment and has no configuration loader; it serves the non-secret snapshot at `GET /core/v1/installation`. Do not add a second operator configuration file, loader precedence, hot reload, fallback to earlier setting names or an embedded Core node. - Names: Core settings use `OAC_*`, Web settings `OAC_WEB_*`, shared Go logging `OAC_LOG_*`. A renamed setting fails startup even when empty or when the new name is also set; report every matching name without its value. Executables are `oac-core`, `oac-core-migrate`, `oac-core-device`, `oac-core-environment-key`, `oac-node`, `oac-web`, and the Core-host E2B helper `oac-e2b-provider` under `/opt/oac/e2b` in the image. The default installation directory is `~/.oac/core`; generated files carry `x-oac` annotations. diff --git a/deploy/install/config.schema.json b/deploy/install/config.schema.json index 4d9937c4f..192221c6b 100644 --- a/deploy/install/config.schema.json +++ b/deploy/install/config.schema.json @@ -83,7 +83,7 @@ "modes": ["all", "web-only"], "restarts": ["web"], "derives": ["Web port mapping or OAC_WEB_ADDR"], - "install_flag": "--port" + "install_flag": "--web-port" } }, "database": { diff --git a/deploy/install/configuration.py b/deploy/install/configuration.py index 0d198aae3..a5582c5c9 100644 --- a/deploy/install/configuration.py +++ b/deploy/install/configuration.py @@ -4,6 +4,7 @@ operator's export headers. Secrets stay in secrets/, one copy each, and reach the services as read-only single-file mounts or file paths. """ +import collections import hashlib import ipaddress import json @@ -17,6 +18,8 @@ # Where Core and Web containers see secrets and generated inputs. RUN = "/run/oac" +# With native Core, PostgreSQL publishes its port here. +DATABASE_HOST = "127.0.0.1" POOL = (("max_conns", "pool_max_conns"), ("min_conns", "pool_min_conns"), ("max_conn_lifetime", "pool_max_conn_lifetime"), ("max_conn_idle_time", "pool_max_conn_idle_time"), ("health_check_period", "pool_health_check_period")) @@ -124,11 +127,15 @@ def loopback_listener(host): return address.is_loopback or bool(mapped and mapped.is_loopback) +def service_host(config, service): + """The address Core or Web listens on; behind managed ingress Core stays on loopback.""" + return str(ipaddress.ip_address("127.0.0.1" if service == "core" and ingress_config.enabled(config) else config["host"])) + + def service_address(config, service, connect=False): """One derivation for listen addresses and local operator connections.""" - host = "127.0.0.1" if service == "core" and ingress_config.enabled(config) else config["host"] + host = service_host(config, service) address = ipaddress.ip_address(host) - host = str(address) if connect and address.is_unspecified: host = "::1" if address.version == 6 else "127.0.0.1" if address.version == 6: @@ -136,6 +143,28 @@ def service_address(config, service, connect=False): return f'{host}:{config["ports"][service]}' +Listener = collections.namedtuple("Listener", "purpose host port setting") + + +def listeners(config): + """Every host listener the services bind, from the addresses they are rendered with. + + setting is the config.json key that owns the port. + """ + mode, ports, result = config["mode"], config["ports"], [] + if ingress_config.enabled(config): + # The gateway publishes Web's port and the HTTPS ports; Web itself publishes none. + result += [Listener("Web", config["host"], port, "ports.web") if port == ports["web"] else + Listener("HTTPS", config["host"], port, "ingress") for port, _ in ingress_config.published(config)] + elif mode != "core-only": + result.append(Listener("Web", service_host(config, "web"), ports["web"], "ports.web")) + if mode != "web-only": + result.append(Listener("Core", service_host(config, "core"), ports["core"], "ports.core")) + if config.get("native_core"): + result.append(Listener("PostgreSQL", DATABASE_HOST, ports["database"], "ports.database")) + return result + + def service_origin(config, service): return "http://" + service_address(config, service, connect=True) @@ -164,7 +193,7 @@ def core_environment(root, config, state): generated = str(root / "generated") if native else RUN secrets = str(root / "secrets") if native else RUN ports, core = config["ports"], config["core"] - database = f'127.0.0.1:{ports["database"]}' if native else "database:5432" + database = f'{DATABASE_HOST}:{ports["database"]}' if native else "database:5432" query = [("sslmode", "disable")] + [(name, str(core["database_pool"][key])) for key, name in POOL if core["database_pool"][key] is not None] result = { @@ -223,7 +252,7 @@ def compose_config(root, config, state): } doc["volumes"] = {"database": {}} if native: - services["database"]["ports"] = [f'127.0.0.1:{config["ports"]["database"]}:5432'] + services["database"]["ports"] = [f'{DATABASE_HOST}:{config["ports"]["database"]}:5432'] else: mounts = [bind(root / "secrets" / name, f"{RUN}/{name}") for name in ("credential.key", "database.password")] if (root / "native-installers/catalog.json").is_file(): diff --git a/deploy/install/ingress_config.py b/deploy/install/ingress_config.py index 00a8f77a0..766b66d24 100644 --- a/deploy/install/ingress_config.py +++ b/deploy/install/ingress_config.py @@ -107,6 +107,11 @@ def reload(root, document): raise RuntimeError("HTTPS gateway is unavailable; inspect gateway logs and retry") from None +def published(config): + """(host port, gateway port) of each port the gateway publishes on config["host"].""" + return [(config["ports"]["web"], 8080), (80, 80), (443, 443)] + + def services(root, config, state, bind): root = Path(root) identity = f'{state["uid"]}:{state["gid"]}' @@ -116,7 +121,7 @@ def services(root, config, state, bind): if ":" in host: host = "[" + host + "]" gateway = dict(common, command=["caddy", "run", "--config", "/generated/Caddyfile", "--adapter", "caddyfile"], - ports=[f'{host}:{config["ports"]["web"]}:8080', f"{host}:80:80", f"{host}:443:443"], + ports=[f"{host}:{port}:{target}" for port, target in published(config)], environment={"XDG_DATA_HOME": "/data", "XDG_CONFIG_HOME": "/data/config", "NO_PROXY": "core,web,localhost,127.0.0.1,::1", "no_proxy": "core,web,localhost,127.0.0.1,::1"}, diff --git a/deploy/install/install.py b/deploy/install/install.py index e2ae5ebbe..acb02d0d2 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -46,6 +46,7 @@ flag.removeprefix("--").replace("-", "_") for flag in SETTING_ARGUMENTS)) RETIRED_NODE_FLAGS = ("--sandbox-provider and --provider are retired: use --sandbox docker|microsandbox|e2b|none. " "The installer no longer adds this host as a node; add it with Add node on the Nodes page in Web.") +AVOID = 20 # An omitted Core or Web port moves at most this far above its default. DOCKER_RISKS = """Docker sandboxes isolate less than microsandbox, the default: - Containers share the node's kernel, so a container escape reaches the host; microsandbox runs each sandbox in its own microVM. @@ -102,15 +103,6 @@ def verify_bundle(bundle): return manifest -def free_port(port, host): - family = socket.AF_INET6 if ipaddress.ip_address(host).version == 6 else socket.AF_INET - with socket.socket(family) as sock: - try: - sock.bind((host, port)) - except OSError: - raise InstallError(f"Port {port} is already in use; select another port") from None - - def database_port(): with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) @@ -241,6 +233,50 @@ def seed_config(args, document): return config_model.initial(mode, native, **values) +def check_listeners(args, document, config): + """Check every listener of a new installation, before anything slow runs. + + A taken port that the flags or the --config file set fails, and so does one that a + loopback public_url names. An omitted Core or Web port moves to the first free port + above its default that no other listener uses. Returns the config and + (purpose, taken port, chosen port) for each move. + """ + if document is None: + names, where = {key: flag for flag, (key, _) in SETTING_ARGUMENTS.items()}, "" + given = {key for key, flag in names.items() if getattr(args, flag.removeprefix("--").replace("-", "_")) is not None} + else: + names, where = {}, " in the --config file" + given = {key for key in ("ports.core", "ports.web") if config_model.lookup(document, key) is not None} + if not oac_cli.address_available(config["host"]): + raise InstallError(f"{config['host']} ({names.get('host', 'host')}{where}) is not an address of this machine; " + "use one of its addresses") + public_url, moved = config["public_url"], [] + for listener in configuration.listeners(config): + if oac_cli.port_free(listener.host, listener.port): + continue + if listener.purpose == "HTTPS": + remedy = "--ingress external" if document is None else '"ingress": "external" in the --config file' + raise InstallError(f"Automatic HTTPS needs ports 80 and 443, and port {listener.port} is already in use on " + f"{listener.host}. Free it, or use an existing reverse proxy with {remedy}; " + f"find the process with: sudo ss -ltnp 'sport = :{listener.port}'") + name = names.get(listener.setting, listener.setting) + # Moving the port would leave a loopback public_url pointing at the old one. + pinned = bool(public_url) and loopback_origin(public_url) and origin_port(public_url) == listener.port + if pinned: + name += " and " + names.get("public_url", "public_url") + if listener.setting in given or pinned or listener.purpose not in ("Core", "Web"): + raise InstallError(oac_cli.port_in_use(listener, name + where)) + taken = {other.port for other in configuration.listeners(config)} + port = next((port for port in range(listener.port + 1, listener.port + AVOID + 1) + if port not in taken and oac_cli.port_free(listener.host, port)), None) + if port is None: + raise InstallError(f"Ports {listener.port} to {listener.port + AVOID} are in use on {listener.host}; " + f"set a free port with {name}{where}") + config["ports"][listener.setting.removeprefix("ports.")] = port + moved.append((listener.purpose, listener.port, port)) + return config, moved + + def loopback_origin(value): hostname = urlsplit(value or "").hostname try: @@ -529,7 +565,7 @@ def create(root, args, config, manifest, images): write(root / "config.json", json.dumps(config, indent=2) + "\n") -def finish(root, bundle, manifest, fresh=False, selection=None): +def finish(root, bundle, manifest, fresh=False, selection=None, moved=()): """Repair and start this release while the installer holds the installation lock.""" state = oac_cli.load_state(root) step("Preparing service files") @@ -563,13 +599,13 @@ def finish(root, bundle, manifest, fresh=False, selection=None): deployment = sandbox_setup.initialize(root, config, state, selection) except sandbox_setup.SandboxSetupError as error: failure = error - summary(root, config, fresh, selection, deployment, incomplete=failure is not None) + summary(root, config, fresh, selection, deployment, incomplete=failure is not None, moved=moved) if failure: raise InstallError(f"{str(failure).rstrip('.')}. Services are installed and running; " f"choose the sandbox backend {choose_where(mode)}") -def summary(root, config, fresh, selection=None, deployment=None, incomplete=False): +def summary(root, config, fresh, selection=None, deployment=None, incomplete=False, moved=()): mode, public_url, ports = config["mode"], config["public_url"], config["ports"] addresses = [] if mode != "core-only": @@ -587,7 +623,7 @@ def summary(root, config, fresh, selection=None, deployment=None, incomplete=Fal # The loopback Web port does not serve the public API. addresses.append("API base URL: " + api + " (local only)") install_output.summary(root, config, addresses, fresh, selection, deployment, - nodes_reach(public_url), incomplete) + nodes_reach(public_url), incomplete, moved) def main(argv=None): @@ -596,17 +632,18 @@ def main(argv=None): if root.is_symlink() or root.resolve() != root: raise InstallError("Installation directory must be canonical and not a symlink") bundle = Path(__file__).resolve().parent - step("Verifying installation files") - manifest = verify_bundle(bundle) - # Refuse foreign state before even creating a lock; repeat under the lock to - # protect against another current installer finishing between these reads. - check_release(root, manifest) if not args.explicit_install_dir: old = Path.home() / ".parsar/core" if (old / "state.json").exists() or (old / "installation.json").exists(): raise InstallError(oac_cli.UNSUPPORTED_VERSION) + # Settings and listeners take seconds to check, so they come before hashing the bundle. step("Checking installation settings") prepared = prepare_fresh(args) if layout(root) == "empty" else None + step("Verifying installation files") + manifest = verify_bundle(bundle) + # Refuse foreign state before even creating a lock; repeat under the lock to + # protect against another current installer finishing between these reads. + check_release(root, manifest) if root.parent == Path.home() / ".oac": root.parent.mkdir(mode=0o700, parents=True, exist_ok=True) root.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -631,6 +668,7 @@ def prepare_fresh(args): choice = check_flags(args, document) config = seed_config(args, document) check_public_url(config, choice) + config, moved = check_listeners(args, document, config) if config["mode"] == "web-only": key = read_core_key_file(args.core_key_file) if oac_cli.core_installation(config["web"]["core_url"], key)[0] == 404: @@ -641,7 +679,7 @@ def prepare_fresh(args): if choice == "e2b" else None) if choice == "docker": confirm_docker(args.accept_docker_risks) - return config, choice, e2b + return config, choice, e2b, moved def install_locked(args, root, bundle, manifest, prepared): @@ -672,22 +710,18 @@ def install_locked(args, root, bundle, manifest, prepared): "service. Preserve the directory and reinstall into a new empty directory") if kind == "other": raise InstallError("Installation directory is not empty; refusing to overwrite existing state") - config, choice, e2b = prepared + config, choice, e2b, moved = prepared step("Checking host requirements") check_host() - manifest = verify_bundle(bundle) if config.get("native_core"): native_service.preflight(bundle, root) - for key in ("core", "web", "database"): - if key in config["ports"]: - free_port(config["ports"][key], "127.0.0.1" if key == "database" else config["host"]) if ingress_config.enabled(config): ingress_config.preflight() selection = None if choice == "none" else sandbox_setup.selection(bundle, manifest, choice, e2b) images = image_loader(manifest, bundle)(image_names(config["mode"], config.get("native_core", False), ingress_config.enabled(config))) step("Creating installation settings and credentials") create(root, args, config, manifest, images) - finish(root, bundle, manifest, fresh=True, selection=selection) + finish(root, bundle, manifest, fresh=True, selection=selection, moved=moved) if __name__ == "__main__": diff --git a/deploy/install/install_output.py b/deploy/install/install_output.py index c2d35d0af..cf9adcaef 100644 --- a/deploy/install/install_output.py +++ b/deploy/install/install_output.py @@ -38,7 +38,7 @@ def sandbox_lines(config, selection, deployment, reachable): return lines -def summary(root, config, addresses, fresh, selection, deployment, reachable, incomplete): +def summary(root, config, addresses, fresh, selection, deployment, reachable, incomplete, moved=()): mode = config["mode"] status = ("Services are running; sandbox setup needs attention." if incomplete else "Installation complete." if fresh else "Installation settings checked. Use Status below to inspect service health.") @@ -46,6 +46,8 @@ def summary(root, config, addresses, fresh, selection, deployment, reachable, in heading("Access") for address in addresses: print(" " + address) + for purpose, taken, port in moved: + print(f" Port {taken} was in use; {purpose} uses {port}.") heading("Sign in" if mode != "core-only" else "Authentication") print(f" Core key file: {root / 'secrets/core.key'}") if mode != "core-only": diff --git a/deploy/install/installer_fakes.py b/deploy/install/installer_fakes.py index c6690ac95..065e81c34 100644 --- a/deploy/install/installer_fakes.py +++ b/deploy/install/installer_fakes.py @@ -6,7 +6,9 @@ starts. systemd runs the unit it last loaded; its process keeps the environment it started with. """ +import errno import hashlib +import ipaddress import json from pathlib import Path import re @@ -47,8 +49,13 @@ def __init__(self, test): self.deployment = {"provider": "", "generation": 0, "reset": None, "resources": {"allocations": 0, "pending": 0}} # what sandbox_setup reads and posts self.deployment_posts = [] self.deployment_refusal = None # Core's message when it refuses the POST + self.busy = set() # (address, port) of every listening socket, the installation's own included + self.unassigned = set() # addresses this host does not have for target, name, value in ((subprocess, "run", mock.Mock(side_effect=self.run)), (oac_cli, "http", self.http), + (oac_cli, "bind_error", self.bind_error), + (oac_cli, "tcp_listeners", lambda: [(ipaddress.ip_address(host), port) + for host, port in self.busy]), (oac_cli, "time", SimpleNamespace(sleep=lambda seconds: None)), (native_service, "_process_environment", self.process_environment), (sandbox_setup, "send", self.sandbox_send)): @@ -56,6 +63,13 @@ def __init__(self, test): patcher.start() test.addCleanup(patcher.stop) + def bind_error(self, host, port): + if host in self.unassigned: + return errno.EADDRNOTAVAIL + address = ipaddress.ip_address(host) + return errno.EADDRINUSE if any(held == port and oac_cli.overlaps(address, ipaddress.ip_address(other)) + for other, held in self.busy) else 0 + def running(self): return {name for name, item in self.containers.items() if item["running"]} @@ -325,6 +339,5 @@ def run_installer(install, bundle, argv): with mock.patch.object(install, "__file__", str(bundle / "install.py")), \ mock.patch.object(install.platform, "system", return_value="Linux"), \ mock.patch.object(install.platform, "machine", return_value="x86_64"), \ - mock.patch.object(install.native_service.platform, "system", return_value="Linux"), \ - mock.patch.object(install, "free_port"): + mock.patch.object(install.native_service.platform, "system", return_value="Linux"): return install.main([str(item) for item in argv]) diff --git a/deploy/install/oac_cli.py b/deploy/install/oac_cli.py index 433ce65c1..64c76ccf9 100644 --- a/deploy/install/oac_cli.py +++ b/deploy/install/oac_cli.py @@ -10,11 +10,14 @@ import argparse import contextlib import datetime +import errno import fcntl +import ipaddress import json import os from pathlib import Path import secrets +import socket import stat import subprocess import sys @@ -180,6 +183,72 @@ def observe(state): return result +def tcp_listeners(): + """(address, port) of each listening TCP socket in this network namespace.""" + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + lines = Path(table).read_text().splitlines()[1:] + except OSError: + continue + for line in lines: + fields = line.split() + if fields[3] != "0A": # TCP_LISTEN + continue + raw, _, port = fields[1].partition(":") + # The kernel prints each 32-bit word of the address in host byte order, little-endian on amd64. + packed = b"".join(bytes.fromhex(raw[index:index + 8])[::-1] for index in range(0, len(raw), 8)) + yield ipaddress.ip_address(packed), int(port, 16) + + +def overlaps(first, second): + if first.version != second.version: + # Of two families, only a dual-stack IPv6 wildcard also takes IPv4 addresses. + return (first if first.version == 6 else second).is_unspecified + return first.is_unspecified or second.is_unspecified or first == second + + +def bind_error(host, port): + """The errno of binding host:port as the services do, or 0 when the bind succeeds. + + SO_REUSEADDR, which Go and Docker listeners set, lets connections in TIME_WAIT pass. + """ + try: + with socket.socket(socket.AF_INET6 if ipaddress.ip_address(host).version == 6 else socket.AF_INET) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind((host, port)) + except OSError as error: + return error.errno + return 0 + + +def address_available(host): + """Whether host is an address of this machine or a wildcard, so services can bind to it.""" + return bind_error(host, 0) not in (errno.EADDRNOTAVAIL, errno.EAFNOSUPPORT) + + +def port_free(host, port, own=()): + """Whether a listener could bind host:port now. + + own holds this installation's (address, port) listeners, which do not count. Where + one overlaps host:port, or the account may not bind a port below 1024, the kernel's + table of listening sockets decides instead of a bind. + """ + address = ipaddress.ip_address(host) + own = {(other, held) for other, held in own if held == port and overlaps(address, other)} + if not own: + error = bind_error(host, port) + if error not in (errno.EACCES, errno.EPERM): + return error != errno.EADDRINUSE + return not any(held == port and overlaps(address, other) and (other, held) not in own + for other, held in tcp_listeners()) + + +def port_in_use(listener, name, outcome=""): + """The message for a port that another program holds; name is its flag or config.json key.""" + return (f"Port {listener.port} ({name}) is already in use on {listener.host}.{outcome} Free it or choose another " + f"port; find the process with: sudo ss -ltnp 'sport = :{listener.port}'") + + def stale(actual, desired, will_run): """Services that should run but don't, or run with other inputs.""" return {name for name in will_run if name != "migrate" and ( @@ -415,8 +484,7 @@ def render_now(root, config, state): def old_public_url(root, config, previous, disk, actual): """The public URL things are bound to: Core's own answer, else the written core.env.""" - old_config = {"host": previous["host"], "ingress": previous.get("ingress"), - "ports": {"core": previous["ports.core"]}} if previous else config + old_config = applied_view(config, previous) if previous else config base = core_base(old_config) if actual.get("core", {}).get("running"): status, body = http(base + "/core/v1/installation", bearer(configuration.read_core_key(root))) @@ -512,6 +580,21 @@ def check_paired_core(root, config, state, previous, args, interactive, out): return installation +def check_new_listeners(config, previous): + """Each listener this change adds must be free; this installation's own listeners do not count.""" + if previous is None: + return + applied = applied_view(config, previous) + if config["host"] != applied["host"] and not address_available(config["host"]): + raise OacError(f"{config['host']} (host) is not an address of this machine; use one of its addresses. " + "Nothing was applied.") + own = {(ipaddress.ip_address(listener.host), listener.port) for listener in configuration.listeners(applied)} + for listener in configuration.listeners(config): + if (ipaddress.ip_address(listener.host), listener.port) not in own and \ + not port_free(listener.host, listener.port, own): + raise OacError(port_in_use(listener, listener.setting, " Nothing was applied.")) + + def finish_apply(root, config, state, gateway_document, will_run): managed = ingress_config.enabled(config) and "gateway" in will_run if managed: @@ -548,6 +631,7 @@ def _apply(root, args, discard_edits, start, interactive, out, rollback=True, re raise OacError("\n".join(f"generated/{name} was edited by hand." for name in edited) + "\nPut the change in config.json and run oac apply --discard-edits, which keeps the" " edited copy as generated/.edited-