diff --git a/.changeset/one-alpine-base.md b/.changeset/one-alpine-base.md new file mode 100644 index 00000000..a6044497 --- /dev/null +++ b/.changeset/one-alpine-base.md @@ -0,0 +1,32 @@ +--- +"ftw": minor +--- + +Run core and the updater sidecar on one shared base layer, and move the optimizer off oldstable. + +The updater sidecar was `docker:27-cli`. That image is alpine-based too, but it +is a *different* alpine — its own base layer on its own cadence — so a host +pulled two rootfs blobs and tracked two upgrade streams for what looked like one +distribution. The sidecar now sits on the same `alpine:3.22` tag as core, with +the `docker` CLI and the compose plugin copied straight out of the official CLI +image. That is the same upstream artifact, with the version pinned explicitly +and no package repository added; both are statically linked Go binaries, so +neither depends on the base's libc. + +The third image cannot join them. CVXPY publishes no musllinux wheels — the +newest that exists is 0.4.10 — and neither do clarabel, osqp or ecos, so the +optimizer cannot be built on alpine without compiling a Rust solver and two C++ +solvers from source on every release, arm64 under emulation included. It stays +on Debian slim, but moves from bookworm to trixie, because bookworm is oldstable +and part of the deployment was already off full security support. The container +boundary test now asserts all of this instead of grepping one file for a fixed +string with no error message. + +The optimizer is versioned independently and moves to 1.4.0: its image is a +materially different artifact once the base changes, and its release workflow +verifies that a published image's revision label matches the commit it claims, +so the new base could not ship under the old version number. + +The zoneinfo database is also embedded in the core binary as a fallback. It was +already installed in the image; the point is that a base without it can no +longer silently push `time.Local` to UTC and mis-time price and plan windows. diff --git a/.changeset/resolve-local-device-names.md b/.changeset/resolve-local-device-names.md new file mode 100644 index 00000000..f314bdc4 --- /dev/null +++ b/.changeset/resolve-local-device-names.md @@ -0,0 +1,24 @@ +--- +"ftw": minor +--- + +Resolve device `.local` names over mDNS so devices can be configured by name instead of a DHCP-assigned IP. + +Go never resolves `.local` itself: it hands those names to libc only when cgo is +available, and FTW builds with `CGO_ENABLED=0`, so a configured `zap.local` +became a unicast DNS query to the site router and failed. That was true on every +base image and every libc. FTW now answers those names itself over multicast +DNS, and every driver transport uses it — Modbus TCP, MQTT (driver and Home +Assistant bridge), HTTP (including TLS-pinned clients), WebSocket and raw TCP. + +Resolution happens per dial rather than once at startup, so a device that moves +to a new DHCP lease is found again on the next reconnect without a config edit. +Answers are cached for the record's TTL (clamped to 30–120 s) so reconnect loops +do not flood the LAN, and failures are cached briefly so a device that is still +booting is retried soon. A failed resolution now logs `mDNS resolution failed` +and names the mechanism, instead of surfacing as a generic dial error. + +Only `.local` names take this path; literal IPs and ordinary DNS names dial +exactly as before. mDNS needs multicast reachability, which the Linux Compose +topology has via `network_mode: host`. Under `docker-compose.macos.yml` the +container is bridged, so configure devices by IP there. diff --git a/Dockerfile b/Dockerfile index 6526349c..88db8e93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,10 +36,24 @@ RUN cd go && \ go build -trimpath -ldflags="-s -w -X main.Version=${VERSION}" \ -o /out/ftw-backup ./cmd/ftw-backup # --- Runtime --------------------------------------------------------------- +# alpine:3.22, and Dockerfile.updater now uses the same tag, so the two images +# a host runs side by side share one base layer instead of pulling two. +# +# The binary is CGO_ENABLED=0 and fully static, so this picks the image's +# *userland*, not a libc the process depends on. That is why the choice is +# reversible and why it has no bearing on .local resolution — see +# go/internal/mdnsresolve. +# +# Dockerfile.optimizer cannot join this base: CVXPY publishes no musllinux +# wheels at all, so see the note there before trying to unify the third image. FROM alpine:3.22 # HTTPS integrations and timezone-aware price/plan windows need these at # runtime. BusyBox wget provides the health check without adding Python/curl. +# +# tzdata is belt-and-braces now: the binary also embeds the zoneinfo database +# (see the time/tzdata import in cmd/ftw). Without either, time.Local silently +# degrades to UTC and mis-times price and plan windows with no error at all. RUN apk add --no-cache ca-certificates tzdata # Image layout: diff --git a/Dockerfile.optimizer b/Dockerfile.optimizer index f8847321..3950a39c 100644 --- a/Dockerfile.optimizer +++ b/Dockerfile.optimizer @@ -1,11 +1,28 @@ # Independently releasable FTW mathematical optimizer. -FROM python:3.12-slim-bookworm AS build +# +# This image does NOT share a base with core and the updater, and cannot. +# Measured, not assumed, on python:3.12-alpine: +# +# pip install --only-binary=:all: cvxpy==1.9.2 +# → "Could not find a version that satisfies the requirement +# cvxpy==1.9.2 (from versions: 0.4.10)" +# +# CVXPY publishes no musllinux wheels — 0.4.10 is the newest that exists — and +# clarabel, osqp and ecos have none either. numpy, scipy and highspy do. Falling +# back to source builds means compiling a Rust solver and two C++ solvers on +# every release, for arm64 under emulation as well; attempted, and it fails in +# sparsediffpy's build backend before it even reaches them. +# +# So the stack runs two bases by necessity. What is fixed here is the suite: +# bookworm is oldstable, so this moves to trixie to match core's security +# cadence even though the layers stay separate. +FROM python:3.12-slim-trixie AS build COPY optimizer/ /src/optimizer/ RUN python -m venv /opt/venv && \ /opt/venv/bin/pip install --no-cache-dir /src/optimizer -FROM python:3.12-slim-bookworm +FROM python:3.12-slim-trixie ARG VERSION=dev ARG BUILD_SHA="" diff --git a/Dockerfile.updater b/Dockerfile.updater index e00b5d5c..50ff1139 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -28,12 +28,29 @@ RUN cd go && \ -o /out/ftw-updater ./cmd/ftw-updater # --- Runtime --------------------------------------------------------------- -# docker:27-cli bundles the `docker` binary + `docker compose` plugin + -# ca-certificates + tzdata, which is exactly what we need to execute -# compose pulls against the host daemon. -FROM docker:27-cli +# The same alpine:3.22 rootfs as Dockerfile, so core and the sidecar share one +# base layer: a host pulls it once instead of twice, and there is one suite to +# track for both. +# +# This was docker:27-cli. That image is alpine-based too, but it is a +# *different* alpine — its own base layer, its own upgrade cadence — so it +# shared nothing with core in practice. +# +# The docker CLI and the compose plugin are copied straight out of the official +# CLI image instead: same upstream artifact, version pinned explicitly, and no +# package repository added. Both are statically linked Go binaries (verified: +# musl's loader reports "Not a valid dynamic program", and both run correctly +# once copied), so nothing here depends on which libc the base ships. +FROM alpine:3.22 + +# docker:27-cli bundled these; on a bare base they have to be asked for. +# Compose pull hits GHCR over TLS. +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker:27-cli /usr/local/libexec/docker/cli-plugins/docker-compose \ + /usr/local/libexec/docker/cli-plugins/docker-compose -# Compose pull hits GHCR over TLS; docker:cli already bundles ca-certs. COPY --from=builder /out/ftw-updater /usr/local/bin/ftw-updater COPY LICENSE NOTICE /usr/share/doc/ftw/ diff --git a/config.example.yaml b/config.example.yaml index 0999020f..ff047809 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -91,7 +91,7 @@ drivers: # battery_telemetry_only: true # capabilities: # http: - # allowed_hosts: ["zap.local"] # use the LAN IP if mDNS is unavailable + # allowed_hosts: ["zap.local"] # .local is resolved by FTW over mDNS # config: # host: zap.local # # meter_serial: p1m-... # optional; P1 is auto-selected diff --git a/docs/sourceful-zap.md b/docs/sourceful-zap.md index 75f98d60..9b6a1306 100644 --- a/docs/sourceful-zap.md +++ b/docs/sourceful-zap.md @@ -76,6 +76,12 @@ go test ./internal/drivers -run 'Zap|zap' - not found: confirm Zap is on Wi-Fi and reachable at `http://zap.local/api/system` from the FTW host; +- `.local` name does not resolve: FTW resolves `.local` itself over multicast + DNS rather than through the OS resolver, so it needs to be on the same L2 + segment as the device. That is the case with the Linux Compose topology + (`network_mode: host`); under `docker-compose.macos.yml` the container is + bridged and multicast does not reach the LAN, so configure the device by IP + there. The log line naming the failure is `mDNS resolution failed`; - no meter: inspect Zap's `/api/devices` and pin `meter_serial` when needed; - duplicate PV/battery: disable the overlapping Zap DER; - visible battery is not controlled: expected for the telemetry-only driver. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..cfe68949 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -24,6 +24,13 @@ import ( "sync" "syscall" "time" + // Embed the zoneinfo database in the binary. Production code uses + // time.Local, which silently falls back to UTC when the runtime has no + // zoneinfo tree — no error, no log line, just price windows and plan + // boundaries an hour or two out. The system database still wins when it is + // present; this only removes the silent-failure mode if a base image ever + // stops shipping tzdata. + _ "time/tzdata" "github.com/srcfl/ftw/go/internal/api" "github.com/srcfl/ftw/go/internal/arp" diff --git a/go/internal/drivers/lua.go b/go/internal/drivers/lua.go index bb62185c..f9cdc9d5 100644 --- a/go/internal/drivers/lua.go +++ b/go/internal/drivers/lua.go @@ -69,6 +69,8 @@ import ( "time" lua "github.com/yuin/gopher-lua" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // LuaDriver wraps a running Lua VM bound to a HostEnv. @@ -1150,8 +1152,16 @@ func registerHost(L *lua.LState, env *HostEnv) { return false, fmt.Sprintf("host %q (port %s) not in allowed_hosts", host, port) } + // Drivers routinely address a device by its ".local" name, which the + // stdlib resolver cannot answer. Clone the default transport so proxying, + // HTTP/2 and connection pooling are all unchanged — only the dial step + // differs, and only for ".local" hosts. + transport := net_http.DefaultTransport.(*net_http.Transport).Clone() + transport.DialContext = mdnsresolve.DialContext + httpClient := &net_http.Client{ - Timeout: 15 * time.Second, + Timeout: 15 * time.Second, + Transport: transport, CheckRedirect: func(req *net_http.Request, via []*net_http.Request) error { if len(via) >= 10 { return fmt.Errorf("stopped after 10 redirects") @@ -1180,7 +1190,10 @@ func registerHost(L *lua.LState, env *HostEnv) { // CA. Drivers WITHOUT a pin keep Go's default transport untouched, so // nothing about existing HTTP drivers changes. if pin := tlsPin; pin != "" { - tr := net_http.DefaultTransport.(*net_http.Transport).Clone() + // Clone the transport built above so the pinned client keeps the same + // mDNS-aware dialer — a pinned device is usually a local appliance + // addressed by its ".local" name, which is exactly the case that needs it. + tr := transport.Clone() tr.TLSClientConfig = &tls.Config{ // We replace chain/hostname verification with our own exact // fingerprint check below, so the stdlib check must be off. diff --git a/go/internal/drivers/tcp_cap.go b/go/internal/drivers/tcp_cap.go index 4153f3cc..cba42622 100644 --- a/go/internal/drivers/tcp_cap.go +++ b/go/internal/drivers/tcp_cap.go @@ -6,6 +6,8 @@ import ( "strings" "sync" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // TCPCap is the host's raw TCP socket capability. One driver = one upstream @@ -93,7 +95,7 @@ func (n *netTCP) Open(addr string) error { return fmt.Errorf("tcp: %s", reason) } - conn, err := net.DialTimeout("tcp", addr, 10*time.Second) + conn, err := mdnsresolve.DialTimeout("tcp", addr, 10*time.Second) if err != nil { return fmt.Errorf("tcp dial: %w", err) } diff --git a/go/internal/drivers/ws_cap.go b/go/internal/drivers/ws_cap.go index 979d5685..bd23736d 100644 --- a/go/internal/drivers/ws_cap.go +++ b/go/internal/drivers/ws_cap.go @@ -9,6 +9,8 @@ import ( "time" "github.com/gorilla/websocket" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // gorillaWS is the production WSCap implementation. One per driver. @@ -73,6 +75,8 @@ func (g *gorillaWS) Open(url string, headers map[string]string) error { } dialer := *websocket.DefaultDialer dialer.HandshakeTimeout = 15 * time.Second + // ".local" hosts need mDNS; everything else falls through to a plain dial. + dialer.NetDialContext = mdnsresolve.DialContext if len(subprotocols) > 0 { dialer.Subprotocols = subprotocols } diff --git a/go/internal/ha/bridge.go b/go/internal/ha/bridge.go index c39e6f57..dd05fd77 100644 --- a/go/internal/ha/bridge.go +++ b/go/internal/ha/bridge.go @@ -12,6 +12,8 @@ import ( "encoding/json" "fmt" "log/slog" + "net" + "net/url" "sort" "strconv" "strings" @@ -22,6 +24,7 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/mdnsresolve" "github.com/srcfl/ftw/go/internal/telemetry" ) @@ -249,6 +252,13 @@ func (b *Bridge) connectAndStart(cfg *config.HomeAssistant, driverNames []string opts := paho.NewClientOptions(). AddBroker(fmt.Sprintf("tcp://%s:%d", cfg.Broker, cfg.Port)). + // A Home Assistant broker is very often reached as homeassistant.local, + // which the stdlib resolver cannot answer. See internal/mqtt for why a + // TCP-only replacement is complete here. + SetCustomOpenConnectionFn(func(uri *url.URL, o paho.ClientOptions) (net.Conn, error) { + d := mdnsresolve.Dialer{Dialer: net.Dialer{Timeout: o.ConnectTimeout}} + return d.Dial("tcp", uri.Host) + }). SetClientID("forty-two-watts-ha"). SetAutoReconnect(true). SetConnectRetry(true). diff --git a/go/internal/mdnsresolve/mdnsresolve.go b/go/internal/mdnsresolve/mdnsresolve.go new file mode 100644 index 00000000..81eb5451 --- /dev/null +++ b/go/internal/mdnsresolve/mdnsresolve.go @@ -0,0 +1,328 @@ +// Package mdnsresolve resolves RFC 6762 ".local" host names over multicast +// DNS and provides a dialer that uses it. +// +// Go never does this itself. net/conf.go routes a ".local" lookup to libc only +// when cgo is available, and every FTW build sets CGO_ENABLED=0, so the pure Go +// resolver is always selected: it reads /etc/resolv.conf and sends a *unicast* +// query to the site router, which has no idea what "inverter.local" is. That +// holds on every base image and every libc — musl and glibc alike — so +// resolving here is the only portable fix. +// +// Only ".local" names take this path. Literal IPs and ordinary DNS names are +// handed straight to the standard dialer. +package mdnsresolve + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" + "sync" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +// mdnsAddr is the RFC 6762 IPv4 multicast group. A var, not a const, so tests +// can aim a query at a loopback responder. +var mdnsAddr = &net.UDPAddr{IP: net.IPv4(224, 0, 0, 251), Port: 5353} + +// listenPacket opens the ephemeral socket a query is sent from. Replaced in +// tests. It deliberately does NOT bind port 5353: avahi-daemon already owns +// that on the host, and the QU bit below asks responders to reply directly to +// this socket instead. +var listenPacket = func() (*net.UDPConn, error) { + return net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) +} + +// now is swappable so cache-expiry tests do not have to sleep. +var now = time.Now + +const ( + // queryTimeout matches the budget the scanner already uses for its reverse + // lookups — long enough for a sleepy device, short enough that a driver + // dial does not stall a control tick. + queryTimeout = 900 * time.Millisecond + + // classQU is IN with the RFC 6762 unicast-response bit set. + classQU = dnsmessage.Class(0x8001) + + // A responder's TTL is advisory here. The floor stops a device that + // advertises a very short TTL from turning every Modbus reconnect into a + // multicast storm; the ceiling keeps a DHCP move from taking effect + // arbitrarily late, which is the whole point of binding by name. + minTTL = 30 * time.Second + maxTTL = 120 * time.Second + + // negativeTTL is deliberately short: a device that was off when we first + // looked should become reachable soon after it boots. + negativeTTL = 5 * time.Second +) + +type cacheEntry struct { + addrs []netip.Addr // empty means a cached negative answer + expires time.Time +} + +var ( + cacheMu sync.Mutex + cache = map[string]cacheEntry{} +) + +// IsLocal reports whether host is a ".local" name that mDNS should resolve. +// A literal IP is never one, so a configured "192.168.1.5" keeps the plain +// dial path and never touches the network for resolution. +func IsLocal(host string) bool { + if host == "" || net.ParseIP(host) != nil { + return false + } + return strings.HasSuffix(strings.ToLower(strings.TrimSuffix(host, ".")), ".local") +} + +func canonical(name string) string { + return strings.ToLower(strings.TrimSuffix(name, ".")) +} + +func cacheLookup(key string) ([]netip.Addr, bool) { + cacheMu.Lock() + defer cacheMu.Unlock() + entry, ok := cache[key] + if !ok || now().After(entry.expires) { + return nil, false + } + return entry.addrs, true +} + +func cacheStore(key string, addrs []netip.Addr, ttl time.Duration) { + cacheMu.Lock() + defer cacheMu.Unlock() + cache[key] = cacheEntry{addrs: addrs, expires: now().Add(ttl)} +} + +// Flush drops every cached answer. Tests use it; nothing in production does. +func Flush() { + cacheMu.Lock() + defer cacheMu.Unlock() + cache = map[string]cacheEntry{} +} + +// Lookup resolves a ".local" name to its advertised addresses. +func Lookup(ctx context.Context, name string) ([]netip.Addr, error) { + key := canonical(name) + if addrs, ok := cacheLookup(key); ok { + if len(addrs) == 0 { + return nil, fmt.Errorf("no mDNS responder for %s (cached)", name) + } + return addrs, nil + } + + addrs, ttl, err := queryAddrs(ctx, key) + if err != nil || len(addrs) == 0 { + cacheStore(key, nil, negativeTTL) + if err == nil { + err = fmt.Errorf("no mDNS responder for %s", name) + } + return nil, err + } + + cacheStore(key, addrs, ttl) + // Logged on a cache miss only, so this is at most one line per TTL per + // device rather than one per reconnect. + slog.Info("resolved host over mDNS", "host", key, "addr", addrs[0].String(), "ttl", ttl) + return addrs, nil +} + +func queryAddrs(ctx context.Context, name string) ([]netip.Addr, time.Duration, error) { + qname, err := dnsmessage.NewName(name + ".") + if err != nil { + return nil, 0, fmt.Errorf("mdns: bad name %q: %w", name, err) + } + // One packet, two questions. RFC 6762 §5.2 allows it and it saves a round + // trip on dual-stack devices. + msg := dnsmessage.Message{Questions: []dnsmessage.Question{ + {Name: qname, Type: dnsmessage.TypeA, Class: classQU}, + {Name: qname, Type: dnsmessage.TypeAAAA, Class: classQU}, + }} + packed, err := msg.Pack() + if err != nil { + return nil, 0, fmt.Errorf("mdns: pack query: %w", err) + } + + var ( + addrs []netip.Addr + ttl time.Duration + ) + err = exchange(ctx, packed, func(packet []byte) bool { + got, gotTTL, ok := parseAddrAnswer(packet, name+".") + if !ok { + return false + } + addrs, ttl = got, gotTTL + return true + }) + if err != nil { + return nil, 0, err + } + return addrs, ttl, nil +} + +// exchange sends one multicast query and feeds every reply to handle until it +// accepts one or the deadline passes. +func exchange(ctx context.Context, packed []byte, handle func([]byte) bool) error { + conn, err := listenPacket() + if err != nil { + return fmt.Errorf("mdns: open socket: %w", err) + } + defer conn.Close() + + deadline := now().Add(queryTimeout) + if d, ok := ctx.Deadline(); ok && d.Before(deadline) { + deadline = d + } + if err := conn.SetDeadline(deadline); err != nil { + return fmt.Errorf("mdns: set deadline: %w", err) + } + if _, err := conn.WriteToUDP(packed, mdnsAddr); err != nil { + return fmt.Errorf("mdns: send query: %w", err) + } + + buf := make([]byte, 1500) + for { + n, _, err := conn.ReadFromUDP(buf) + if err != nil { + return fmt.Errorf("mdns: no usable answer: %w", err) + } + if handle(buf[:n]) { + return nil + } + } +} + +func parseAddrAnswer(packet []byte, qname string) ([]netip.Addr, time.Duration, bool) { + var p dnsmessage.Parser + if _, err := p.Start(packet); err != nil { + return nil, 0, false + } + if err := p.SkipAllQuestions(); err != nil { + return nil, 0, false + } + var addrs []netip.Addr + ttl := maxTTL + // Labelled so a parse error inside the type switch abandons the whole + // packet: once the parser desynchronises, every later record is suspect. +parse: + for { + h, err := p.AnswerHeader() + if err != nil { + break parse + } + if !strings.EqualFold(h.Name.String(), qname) { + if err := p.SkipAnswer(); err != nil { + break parse + } + continue + } + switch h.Type { + case dnsmessage.TypeA: + r, err := p.AResource() + if err != nil { + break parse + } + addrs = append(addrs, netip.AddrFrom4(r.A)) + case dnsmessage.TypeAAAA: + r, err := p.AAAAResource() + if err != nil { + break parse + } + // Unmap so a v4-mapped AAAA dials as plain IPv4. + addrs = append(addrs, netip.AddrFrom16(r.AAAA).Unmap()) + default: + if err := p.SkipAnswer(); err != nil { + break parse + } + continue + } + if d := time.Duration(h.TTL) * time.Second; d < ttl { + ttl = d + } + } + return finishAnswer(addrs, ttl) +} + +func finishAnswer(addrs []netip.Addr, ttl time.Duration) ([]netip.Addr, time.Duration, bool) { + if len(addrs) == 0 { + return nil, 0, false + } + switch { + case ttl < minTTL: + ttl = minTTL + case ttl > maxTTL: + ttl = maxTTL + } + return addrs, ttl, true +} + +// Dialer dials TCP addresses, resolving ".local" host names over mDNS first. +// The embedded net.Dialer carries timeout and keep-alive; anything that is not +// a ".local" name is handed straight to it. +// +// Resolution happens per dial, not once at startup. That is what makes binding +// a device by name survive a DHCP lease change: callers that rebuild their +// connection from the original address string pick up the new IP on reconnect. +type Dialer struct { + net.Dialer +} + +// DialContext resolves address if it names a ".local" host, then dials it. +func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil || !IsLocal(host) { + return d.Dialer.DialContext(ctx, network, address) + } + + addrs, err := Lookup(ctx, host) + if err != nil { + // Name the mechanism. Without this the operator sees a bare dial + // failure and has no way to tell that resolution was the reason. + slog.Warn("mDNS resolution failed; check the device is on this LAN and the container uses host networking", + "host", host, "err", err) + return nil, fmt.Errorf("resolve %s over mDNS: %w", host, err) + } + + var firstErr error + for _, a := range addrs { + conn, err := d.Dialer.DialContext(ctx, network, net.JoinHostPort(a.String(), port)) + if err == nil { + return conn, nil + } + if firstErr == nil { + firstErr = err + } + } + return nil, fmt.Errorf("dial %s over mDNS: %w", host, firstErr) +} + +// Dial is the context-free form, for callers that have no context to pass. +func (d *Dialer) Dial(network, address string) (net.Conn, error) { + ctx := context.Background() + if d.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, d.Timeout) + defer cancel() + } + return d.DialContext(ctx, network, address) +} + +// DialContext dials with default settings. +func DialContext(ctx context.Context, network, address string) (net.Conn, error) { + var d Dialer + return d.DialContext(ctx, network, address) +} + +// DialTimeout mirrors net.DialTimeout with mDNS resolution added. +func DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + d := Dialer{Dialer: net.Dialer{Timeout: timeout}} + return d.Dial(network, address) +} diff --git a/go/internal/mdnsresolve/mdnsresolve_test.go b/go/internal/mdnsresolve/mdnsresolve_test.go new file mode 100644 index 00000000..fb9469a5 --- /dev/null +++ b/go/internal/mdnsresolve/mdnsresolve_test.go @@ -0,0 +1,265 @@ +package mdnsresolve + +import ( + "context" + "errors" + "net" + "net/netip" + "strings" + "testing" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +func TestIsLocal(t *testing.T) { + cases := []struct { + host string + want bool + }{ + {"inverter.local", true}, + {"INVERTER.LOCAL", true}, + {"inverter.local.", true}, + {"zap.local", true}, + // A literal address must never trigger a multicast query. + {"192.168.1.5", false}, + {"::1", false}, + {"example.com", false}, + {"localhost", false}, + {"local", false}, + {"notlocal", false}, + {"", false}, + } + for _, c := range cases { + if got := IsLocal(c.host); got != c.want { + t.Errorf("IsLocal(%q) = %v, want %v", c.host, got, c.want) + } + } +} + +func aResource(t *testing.T, name string, ip [4]byte, ttl uint32) dnsmessage.Resource { + t.Helper() + return dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: mustDNSName(t, name), Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, TTL: ttl, + }, + Body: &dnsmessage.AResource{A: ip}, + } +} + +func packAnswer(t *testing.T, qname string, answers []dnsmessage.Resource) []byte { + t.Helper() + msg := dnsmessage.Message{ + Header: dnsmessage.Header{Response: true, Authoritative: true}, + Questions: []dnsmessage.Question{{Name: mustDNSName(t, qname), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, + Answers: answers, + } + packet, err := msg.Pack() + if err != nil { + t.Fatalf("pack: %v", err) + } + return packet +} + +func TestParseAddrAnswer(t *testing.T) { + qname := "inverter.local." + packet := packAnswer(t, qname, []dnsmessage.Resource{aResource(t, qname, [4]byte{192, 168, 1, 42}, 60)}) + + addrs, ttl, ok := parseAddrAnswer(packet, qname) + if !ok { + t.Fatal("parseAddrAnswer did not accept a valid answer") + } + if len(addrs) != 1 || addrs[0].String() != "192.168.1.42" { + t.Fatalf("addrs = %v, want [192.168.1.42]", addrs) + } + if ttl != 60*time.Second { + t.Fatalf("ttl = %v, want 60s", ttl) + } + + // An answer for a different name must be ignored. + if _, _, ok := parseAddrAnswer(packet, "other.local."); ok { + t.Fatal("accepted an answer for a different name") + } + // Garbage must not panic or resolve. + if _, _, ok := parseAddrAnswer([]byte{1, 2, 3}, qname); ok { + t.Fatal("accepted a malformed packet") + } +} + +func TestParseAddrAnswerClampsTTL(t *testing.T) { + qname := "inverter.local." + for _, c := range []struct { + name string + ttl uint32 + want time.Duration + }{ + // A device advertising a 1 s TTL must not make every Modbus reconnect + // re-query the LAN. + {"below floor", 1, minTTL}, + // A very long TTL must not outlive a DHCP move. + {"above ceiling", 86400, maxTTL}, + {"inside range", 90, 90 * time.Second}, + } { + t.Run(c.name, func(t *testing.T) { + packet := packAnswer(t, qname, []dnsmessage.Resource{aResource(t, qname, [4]byte{10, 0, 0, 1}, c.ttl)}) + _, ttl, ok := parseAddrAnswer(packet, qname) + if !ok { + t.Fatal("answer rejected") + } + if ttl != c.want { + t.Fatalf("ttl = %v, want %v", ttl, c.want) + } + }) + } +} + +// startResponder points the package at a loopback UDP socket that answers one +// query, so the real send/parse path is exercised without touching the LAN. +func startResponder(t *testing.T, answers []dnsmessage.Resource) { + t.Helper() + rc, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + if err != nil { + t.Fatalf("listen responder: %v", err) + } + + origAddr, origListen := mdnsAddr, listenPacket + mdnsAddr = rc.LocalAddr().(*net.UDPAddr) + listenPacket = func() (*net.UDPConn, error) { + return net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + } + + done := make(chan struct{}) + go func() { + defer close(done) + buf := make([]byte, 1500) + _ = rc.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, from, err := rc.ReadFromUDP(buf) + if err != nil { + return + } + if answers == nil { + return // silent responder: exercises the negative path + } + var p dnsmessage.Parser + hdr, err := p.Start(buf[:n]) + if err != nil { + return + } + q, err := p.Question() + if err != nil { + return + } + resp := dnsmessage.Message{ + Header: dnsmessage.Header{ID: hdr.ID, Response: true, Authoritative: true}, + Questions: []dnsmessage.Question{q}, + Answers: answers, + } + packed, err := resp.Pack() + if err != nil { + return + } + _, _ = rc.WriteToUDP(packed, from) + }() + + t.Cleanup(func() { + _ = rc.Close() + <-done + mdnsAddr, listenPacket = origAddr, origListen + Flush() + }) +} + +func TestLookupResolvesLocalName(t *testing.T) { + Flush() + startResponder(t, []dnsmessage.Resource{aResource(t, "inverter.local.", [4]byte{192, 168, 1, 42}, 60)}) + + addrs, err := Lookup(context.Background(), "inverter.local") + if err != nil { + t.Fatalf("Lookup: %v", err) + } + if len(addrs) != 1 || addrs[0] != netip.MustParseAddr("192.168.1.42") { + t.Fatalf("addrs = %v, want [192.168.1.42]", addrs) + } + + // The answer must now be cached: a second call cannot need the responder, + // which has already stopped. + again, err := Lookup(context.Background(), "INVERTER.local") + if err != nil { + t.Fatalf("cached Lookup: %v", err) + } + if len(again) != 1 || again[0] != addrs[0] { + t.Fatalf("cached addrs = %v, want %v", again, addrs) + } +} + +func TestLookupCachesNegativeAnswer(t *testing.T) { + Flush() + startResponder(t, nil) // never answers + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if _, err := Lookup(ctx, "missing.local"); err == nil { + t.Fatal("expected a lookup failure when nothing answers") + } + + addrs, ok := cacheLookup("missing.local") + if !ok { + t.Fatal("a failed lookup should be negatively cached") + } + if len(addrs) != 0 { + t.Fatalf("negative cache holds %v, want no addresses", addrs) + } +} + +func TestCacheExpires(t *testing.T) { + Flush() + base := time.Now() + orig := now + now = func() time.Time { return base } + t.Cleanup(func() { now = orig; Flush() }) + + cacheStore("inverter.local", []netip.Addr{netip.MustParseAddr("192.168.1.9")}, 30*time.Second) + if _, ok := cacheLookup("inverter.local"); !ok { + t.Fatal("entry should be live immediately after store") + } + + now = func() time.Time { return base.Add(31 * time.Second) } + if _, ok := cacheLookup("inverter.local"); ok { + t.Fatal("entry should have expired") + } +} + +func TestDialerSkipsResolutionForPlainHosts(t *testing.T) { + orig := listenPacket + listenPacket = func() (*net.UDPConn, error) { + t.Error("issued an mDNS query for a host that is not a .local name") + return nil, errors.New("should not be called") + } + t.Cleanup(func() { listenPacket = orig }) + + d := Dialer{Dialer: net.Dialer{Timeout: 500 * time.Millisecond}} + // Nothing listens on port 1; the point is that the failure comes from the + // dial, not from resolution. + if _, err := d.Dial("tcp", "127.0.0.1:1"); err == nil { + t.Fatal("expected the dial to fail") + } else if strings.Contains(err.Error(), "mDNS") { + t.Fatalf("plain IP dial went through mDNS: %v", err) + } +} + +func TestDialerReportsResolutionFailure(t *testing.T) { + Flush() + startResponder(t, nil) // never answers + + d := Dialer{Dialer: net.Dialer{Timeout: 100 * time.Millisecond}} + _, err := d.Dial("tcp", "missing.local:502") + if err == nil { + t.Fatal("expected a failure") + } + // The error must name the mechanism — an operator reading the log has to be + // able to tell resolution apart from an unreachable device. + if !strings.Contains(err.Error(), "mDNS") { + t.Fatalf("error %q does not mention mDNS", err) + } +} diff --git a/go/internal/scanner/mdns.go b/go/internal/mdnsresolve/reverse.go similarity index 51% rename from go/internal/scanner/mdns.go rename to go/internal/mdnsresolve/reverse.go index 4963ed52..b1206136 100644 --- a/go/internal/scanner/mdns.go +++ b/go/internal/mdnsresolve/reverse.go @@ -1,20 +1,19 @@ -package scanner +package mdnsresolve import ( "context" "fmt" "net" "strings" - "time" "golang.org/x/net/dns/dnsmessage" ) -var mdnsAddr = &net.UDPAddr{IP: net.IPv4(224, 0, 0, 251), Port: 5353} - -// reverseMDNS sends a reverse PTR query with the QU bit set so devices reply -// directly to our ephemeral socket instead of requiring a bind to port 5353. -func reverseMDNS(ctx context.Context, ip string) string { +// ReverseLookup asks the device at ip to name itself, so a discovered device +// can be shown and stored by its self-broadcast ".local" name rather than a +// DHCP-assigned address. Returns "" when nothing answers — callers treat a +// missing name as ordinary, not as an error. +func ReverseLookup(ctx context.Context, ip string) string { v4 := net.ParseIP(ip).To4() if v4 == nil { return "" @@ -25,35 +24,19 @@ func reverseMDNS(ctx context.Context, ip string) string { return "" } msg := dnsmessage.Message{Questions: []dnsmessage.Question{{ - Name: name, Type: dnsmessage.TypePTR, Class: dnsmessage.Class(0x8001), + Name: name, Type: dnsmessage.TypePTR, Class: classQU, }}} packed, err := msg.Pack() if err != nil { return "" } - conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) - if err != nil { - return "" - } - defer conn.Close() - deadline := time.Now().Add(900 * time.Millisecond) - if d, ok := ctx.Deadline(); ok && d.Before(deadline) { - deadline = d - } - _ = conn.SetDeadline(deadline) - if _, err := conn.WriteToUDP(packed, mdnsAddr); err != nil { - return "" - } - buf := make([]byte, 1500) - for { - n, _, err := conn.ReadFromUDP(buf) - if err != nil { - return "" - } - if host := parsePTRAnswer(buf[:n], qname); host != "" { - return host - } - } + + var host string + _ = exchange(ctx, packed, func(packet []byte) bool { + host = parsePTRAnswer(packet, qname) + return host != "" + }) + return host } func parsePTRAnswer(packet []byte, qname string) string { diff --git a/go/internal/scanner/mdns_test.go b/go/internal/mdnsresolve/reverse_test.go similarity index 98% rename from go/internal/scanner/mdns_test.go rename to go/internal/mdnsresolve/reverse_test.go index 5a77c8f6..af8801bc 100644 --- a/go/internal/scanner/mdns_test.go +++ b/go/internal/mdnsresolve/reverse_test.go @@ -1,4 +1,4 @@ -package scanner +package mdnsresolve import ( "testing" diff --git a/go/internal/modbus/tcp_client.go b/go/internal/modbus/tcp_client.go index 78b7e7ba..b5b790ab 100644 --- a/go/internal/modbus/tcp_client.go +++ b/go/internal/modbus/tcp_client.go @@ -7,6 +7,8 @@ import ( "io" "net" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) const ( @@ -39,10 +41,13 @@ func newTCPClient(addr string, timeout, keepAlive time.Duration) *tcpClient { } func (c *tcpClient) Open() error { - dialer := net.Dialer{ + // Resolution happens here, on every Open, so a device configured by its + // ".local" name is found again after a DHCP lease moves it — the reconnect + // path rebuilds the client from c.addr and picks up the new address. + dialer := mdnsresolve.Dialer{Dialer: net.Dialer{ Timeout: modbusDialTimeout, KeepAlive: c.keepAlive, - } + }} conn, err := dialer.Dial("tcp", c.addr) if err != nil { return err diff --git a/go/internal/mqtt/client.go b/go/internal/mqtt/client.go index fb1ec167..a711dd96 100644 --- a/go/internal/mqtt/client.go +++ b/go/internal/mqtt/client.go @@ -4,12 +4,15 @@ package mqtt import ( "fmt" "log/slog" + "net" + "net/url" "sync" "time" paho "github.com/eclipse/paho.mqtt.golang" "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // Capability wraps a paho client to match drivers.MQTTCap. @@ -51,6 +54,14 @@ func Dial(host string, port int, username, password, clientID string) (*Capabili } opts := paho.NewClientOptions(). AddBroker(fmt.Sprintf("tcp://%s:%d", host, port)). + // paho's built-in dialer goes through the stdlib resolver, which never + // answers a ".local" name. Every broker URL built here is tcp://, so a + // TCP-only replacement is complete; non-".local" hosts fall through to + // a plain dial inside mdnsresolve. + SetCustomOpenConnectionFn(func(uri *url.URL, o paho.ClientOptions) (net.Conn, error) { + d := mdnsresolve.Dialer{Dialer: net.Dialer{Timeout: o.ConnectTimeout}} + return d.Dial("tcp", uri.Host) + }). SetClientID(clientID). SetAutoReconnect(true). SetConnectRetry(true). diff --git a/go/internal/scanner/scanner.go b/go/internal/scanner/scanner.go index 0d8dea31..612d8207 100644 --- a/go/internal/scanner/scanner.go +++ b/go/internal/scanner/scanner.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // FoundDevice is one open port discovered on the local network. @@ -129,7 +131,7 @@ func resolveHostnames(ctx context.Context, devices []FoundDevice) { } if name == "" && ctx.Err() == nil { mdnsCtx, cancel := context.WithTimeout(ctx, 900*time.Millisecond) - name = reverseMDNS(mdnsCtx, ip) + name = mdnsresolve.ReverseLookup(mdnsCtx, ip) cancel() } if name != "" { diff --git a/optimizer/pyproject.toml b/optimizer/pyproject.toml index 2611fca7..8ef2a8de 100644 --- a/optimizer/pyproject.toml +++ b/optimizer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ftw-optimizer" -version = "1.3.2" +version = "1.4.0" description = "CVXPY planning engine for ftw" requires-python = ">=3.11" dependencies = [ diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index ea827f6b..ce871a7b 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -9,7 +9,36 @@ if grep -Eq 'COPY optimizer/|--from=optimizer|/opt/venv|FTW_OPTIMIZER_(PYTHON|DI exit 1 fi -grep -q '^FROM alpine:' Dockerfile +# Core and the updater are meant to share one base layer, so a host pulls that +# rootfs once. Comparing the two runtime stages is what actually holds that; +# grepping for a fixed string in one file only ever proved half of it, and did +# so without printing anything when it failed. +core_base=$(grep -E '^FROM ' Dockerfile | tail -1 | awk '{print $2}') +updater_base=$(grep -E '^FROM ' Dockerfile.updater | tail -1 | awk '{print $2}') + +case "${core_base}" in + alpine:*) ;; + *) + echo "Dockerfile runtime stage is '${core_base}', expected an alpine: tag" >&2 + exit 1 + ;; +esac + +if [ "${core_base}" != "${updater_base}" ]; then + echo "core and updater runtime bases differ: '${core_base}' vs '${updater_base}'" >&2 + echo "they must be the same tag or the shared base layer silently stops being shared" >&2 + exit 1 +fi + +# The optimizer is knowingly on a different base: CVXPY publishes no musllinux +# wheels, so it cannot follow the other two onto alpine. Assert that rather than +# leave it looking like an oversight someone should "fix". +if grep -qE '^FROM python:[^ ]*alpine' Dockerfile.optimizer; then + echo "Dockerfile.optimizer is on an alpine base, which cannot resolve CVXPY" >&2 + echo "cvxpy has no musllinux wheel (newest is 0.4.10); see the note in that file" >&2 + exit 1 +fi + grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile