From aea6c6116d5a408d52915e875a02ae19094bd2f2 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Thu, 30 Jul 2026 16:19:19 +0200 Subject: [PATCH 1/2] feat(net): resolve device .local names over mDNS Go never resolves ".local" itself. net/conf.go routes those names to libc only when cgo is available, and every FTW build sets CGO_ENABLED=0, so the pure Go resolver is always selected: a configured "zap.local" became a unicast DNS query to the site router and failed. That holds on every base image and every libc, musl and glibc alike, so the container's distro was never the variable here. Add internal/mdnsresolve, which answers those names over multicast DNS, and route every driver transport through it: Modbus TCP, MQTT (driver and Home Assistant bridge), HTTP including the TLS-pinned client, WebSocket and raw TCP. Only ".local" names take the new path; literal IPs and ordinary DNS names dial exactly as before. Resolution runs per dial rather than once at startup, so a device that moves to a new DHCP lease is found again on the next reconnect with no config edit. Answers are cached for the record TTL clamped to 30-120s so reconnect loops cannot flood the LAN, and failures are cached for 5s so a still-booting device is retried soon. Failures log "mDNS resolution failed" and name the mechanism rather than surfacing as a generic dial error. The scanner's reverse PTR lookup moves into the same package so there is one mDNS implementation instead of two. Co-authored-by: HuggeK <48095810+HuggeK@users.noreply.github.com> --- .changeset/resolve-local-device-names.md | 24 ++ config.example.yaml | 2 +- docs/sourceful-zap.md | 6 + go/internal/drivers/lua.go | 17 +- go/internal/drivers/tcp_cap.go | 4 +- go/internal/drivers/ws_cap.go | 4 + go/internal/ha/bridge.go | 10 + go/internal/mdnsresolve/mdnsresolve.go | 328 ++++++++++++++++++ go/internal/mdnsresolve/mdnsresolve_test.go | 265 ++++++++++++++ .../mdns.go => mdnsresolve/reverse.go} | 45 +-- .../reverse_test.go} | 2 +- go/internal/modbus/tcp_client.go | 9 +- go/internal/mqtt/client.go | 11 + go/internal/scanner/scanner.go | 4 +- 14 files changed, 692 insertions(+), 39 deletions(-) create mode 100644 .changeset/resolve-local-device-names.md create mode 100644 go/internal/mdnsresolve/mdnsresolve.go create mode 100644 go/internal/mdnsresolve/mdnsresolve_test.go rename go/internal/{scanner/mdns.go => mdnsresolve/reverse.go} (51%) rename go/internal/{scanner/mdns_test.go => mdnsresolve/reverse_test.go} (98%) diff --git a/.changeset/resolve-local-device-names.md b/.changeset/resolve-local-device-names.md new file mode 100644 index 00000000..f314bdc4 --- /dev/null +++ b/.changeset/resolve-local-device-names.md @@ -0,0 +1,24 @@ +--- +"ftw": minor +--- + +Resolve device `.local` names over mDNS so devices can be configured by name instead of a DHCP-assigned IP. + +Go never resolves `.local` itself: it hands those names to libc only when cgo is +available, and FTW builds with `CGO_ENABLED=0`, so a configured `zap.local` +became a unicast DNS query to the site router and failed. That was true on every +base image and every libc. FTW now answers those names itself over multicast +DNS, and every driver transport uses it — Modbus TCP, MQTT (driver and Home +Assistant bridge), HTTP (including TLS-pinned clients), WebSocket and raw TCP. + +Resolution happens per dial rather than once at startup, so a device that moves +to a new DHCP lease is found again on the next reconnect without a config edit. +Answers are cached for the record's TTL (clamped to 30–120 s) so reconnect loops +do not flood the LAN, and failures are cached briefly so a device that is still +booting is retried soon. A failed resolution now logs `mDNS resolution failed` +and names the mechanism, instead of surfacing as a generic dial error. + +Only `.local` names take this path; literal IPs and ordinary DNS names dial +exactly as before. mDNS needs multicast reachability, which the Linux Compose +topology has via `network_mode: host`. Under `docker-compose.macos.yml` the +container is bridged, so configure devices by IP there. diff --git a/config.example.yaml b/config.example.yaml index 0999020f..ff047809 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -91,7 +91,7 @@ drivers: # battery_telemetry_only: true # capabilities: # http: - # allowed_hosts: ["zap.local"] # use the LAN IP if mDNS is unavailable + # allowed_hosts: ["zap.local"] # .local is resolved by FTW over mDNS # config: # host: zap.local # # meter_serial: p1m-... # optional; P1 is auto-selected diff --git a/docs/sourceful-zap.md b/docs/sourceful-zap.md index 75f98d60..9b6a1306 100644 --- a/docs/sourceful-zap.md +++ b/docs/sourceful-zap.md @@ -76,6 +76,12 @@ go test ./internal/drivers -run 'Zap|zap' - not found: confirm Zap is on Wi-Fi and reachable at `http://zap.local/api/system` from the FTW host; +- `.local` name does not resolve: FTW resolves `.local` itself over multicast + DNS rather than through the OS resolver, so it needs to be on the same L2 + segment as the device. That is the case with the Linux Compose topology + (`network_mode: host`); under `docker-compose.macos.yml` the container is + bridged and multicast does not reach the LAN, so configure the device by IP + there. The log line naming the failure is `mDNS resolution failed`; - no meter: inspect Zap's `/api/devices` and pin `meter_serial` when needed; - duplicate PV/battery: disable the overlapping Zap DER; - visible battery is not controlled: expected for the telemetry-only driver. diff --git a/go/internal/drivers/lua.go b/go/internal/drivers/lua.go index bb62185c..f9cdc9d5 100644 --- a/go/internal/drivers/lua.go +++ b/go/internal/drivers/lua.go @@ -69,6 +69,8 @@ import ( "time" lua "github.com/yuin/gopher-lua" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // LuaDriver wraps a running Lua VM bound to a HostEnv. @@ -1150,8 +1152,16 @@ func registerHost(L *lua.LState, env *HostEnv) { return false, fmt.Sprintf("host %q (port %s) not in allowed_hosts", host, port) } + // Drivers routinely address a device by its ".local" name, which the + // stdlib resolver cannot answer. Clone the default transport so proxying, + // HTTP/2 and connection pooling are all unchanged — only the dial step + // differs, and only for ".local" hosts. + transport := net_http.DefaultTransport.(*net_http.Transport).Clone() + transport.DialContext = mdnsresolve.DialContext + httpClient := &net_http.Client{ - Timeout: 15 * time.Second, + Timeout: 15 * time.Second, + Transport: transport, CheckRedirect: func(req *net_http.Request, via []*net_http.Request) error { if len(via) >= 10 { return fmt.Errorf("stopped after 10 redirects") @@ -1180,7 +1190,10 @@ func registerHost(L *lua.LState, env *HostEnv) { // CA. Drivers WITHOUT a pin keep Go's default transport untouched, so // nothing about existing HTTP drivers changes. if pin := tlsPin; pin != "" { - tr := net_http.DefaultTransport.(*net_http.Transport).Clone() + // Clone the transport built above so the pinned client keeps the same + // mDNS-aware dialer — a pinned device is usually a local appliance + // addressed by its ".local" name, which is exactly the case that needs it. + tr := transport.Clone() tr.TLSClientConfig = &tls.Config{ // We replace chain/hostname verification with our own exact // fingerprint check below, so the stdlib check must be off. diff --git a/go/internal/drivers/tcp_cap.go b/go/internal/drivers/tcp_cap.go index 4153f3cc..cba42622 100644 --- a/go/internal/drivers/tcp_cap.go +++ b/go/internal/drivers/tcp_cap.go @@ -6,6 +6,8 @@ import ( "strings" "sync" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // TCPCap is the host's raw TCP socket capability. One driver = one upstream @@ -93,7 +95,7 @@ func (n *netTCP) Open(addr string) error { return fmt.Errorf("tcp: %s", reason) } - conn, err := net.DialTimeout("tcp", addr, 10*time.Second) + conn, err := mdnsresolve.DialTimeout("tcp", addr, 10*time.Second) if err != nil { return fmt.Errorf("tcp dial: %w", err) } diff --git a/go/internal/drivers/ws_cap.go b/go/internal/drivers/ws_cap.go index 979d5685..bd23736d 100644 --- a/go/internal/drivers/ws_cap.go +++ b/go/internal/drivers/ws_cap.go @@ -9,6 +9,8 @@ import ( "time" "github.com/gorilla/websocket" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // gorillaWS is the production WSCap implementation. One per driver. @@ -73,6 +75,8 @@ func (g *gorillaWS) Open(url string, headers map[string]string) error { } dialer := *websocket.DefaultDialer dialer.HandshakeTimeout = 15 * time.Second + // ".local" hosts need mDNS; everything else falls through to a plain dial. + dialer.NetDialContext = mdnsresolve.DialContext if len(subprotocols) > 0 { dialer.Subprotocols = subprotocols } diff --git a/go/internal/ha/bridge.go b/go/internal/ha/bridge.go index c39e6f57..dd05fd77 100644 --- a/go/internal/ha/bridge.go +++ b/go/internal/ha/bridge.go @@ -12,6 +12,8 @@ import ( "encoding/json" "fmt" "log/slog" + "net" + "net/url" "sort" "strconv" "strings" @@ -22,6 +24,7 @@ import ( "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/mdnsresolve" "github.com/srcfl/ftw/go/internal/telemetry" ) @@ -249,6 +252,13 @@ func (b *Bridge) connectAndStart(cfg *config.HomeAssistant, driverNames []string opts := paho.NewClientOptions(). AddBroker(fmt.Sprintf("tcp://%s:%d", cfg.Broker, cfg.Port)). + // A Home Assistant broker is very often reached as homeassistant.local, + // which the stdlib resolver cannot answer. See internal/mqtt for why a + // TCP-only replacement is complete here. + SetCustomOpenConnectionFn(func(uri *url.URL, o paho.ClientOptions) (net.Conn, error) { + d := mdnsresolve.Dialer{Dialer: net.Dialer{Timeout: o.ConnectTimeout}} + return d.Dial("tcp", uri.Host) + }). SetClientID("forty-two-watts-ha"). SetAutoReconnect(true). SetConnectRetry(true). diff --git a/go/internal/mdnsresolve/mdnsresolve.go b/go/internal/mdnsresolve/mdnsresolve.go new file mode 100644 index 00000000..81eb5451 --- /dev/null +++ b/go/internal/mdnsresolve/mdnsresolve.go @@ -0,0 +1,328 @@ +// Package mdnsresolve resolves RFC 6762 ".local" host names over multicast +// DNS and provides a dialer that uses it. +// +// Go never does this itself. net/conf.go routes a ".local" lookup to libc only +// when cgo is available, and every FTW build sets CGO_ENABLED=0, so the pure Go +// resolver is always selected: it reads /etc/resolv.conf and sends a *unicast* +// query to the site router, which has no idea what "inverter.local" is. That +// holds on every base image and every libc — musl and glibc alike — so +// resolving here is the only portable fix. +// +// Only ".local" names take this path. Literal IPs and ordinary DNS names are +// handed straight to the standard dialer. +package mdnsresolve + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" + "sync" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +// mdnsAddr is the RFC 6762 IPv4 multicast group. A var, not a const, so tests +// can aim a query at a loopback responder. +var mdnsAddr = &net.UDPAddr{IP: net.IPv4(224, 0, 0, 251), Port: 5353} + +// listenPacket opens the ephemeral socket a query is sent from. Replaced in +// tests. It deliberately does NOT bind port 5353: avahi-daemon already owns +// that on the host, and the QU bit below asks responders to reply directly to +// this socket instead. +var listenPacket = func() (*net.UDPConn, error) { + return net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) +} + +// now is swappable so cache-expiry tests do not have to sleep. +var now = time.Now + +const ( + // queryTimeout matches the budget the scanner already uses for its reverse + // lookups — long enough for a sleepy device, short enough that a driver + // dial does not stall a control tick. + queryTimeout = 900 * time.Millisecond + + // classQU is IN with the RFC 6762 unicast-response bit set. + classQU = dnsmessage.Class(0x8001) + + // A responder's TTL is advisory here. The floor stops a device that + // advertises a very short TTL from turning every Modbus reconnect into a + // multicast storm; the ceiling keeps a DHCP move from taking effect + // arbitrarily late, which is the whole point of binding by name. + minTTL = 30 * time.Second + maxTTL = 120 * time.Second + + // negativeTTL is deliberately short: a device that was off when we first + // looked should become reachable soon after it boots. + negativeTTL = 5 * time.Second +) + +type cacheEntry struct { + addrs []netip.Addr // empty means a cached negative answer + expires time.Time +} + +var ( + cacheMu sync.Mutex + cache = map[string]cacheEntry{} +) + +// IsLocal reports whether host is a ".local" name that mDNS should resolve. +// A literal IP is never one, so a configured "192.168.1.5" keeps the plain +// dial path and never touches the network for resolution. +func IsLocal(host string) bool { + if host == "" || net.ParseIP(host) != nil { + return false + } + return strings.HasSuffix(strings.ToLower(strings.TrimSuffix(host, ".")), ".local") +} + +func canonical(name string) string { + return strings.ToLower(strings.TrimSuffix(name, ".")) +} + +func cacheLookup(key string) ([]netip.Addr, bool) { + cacheMu.Lock() + defer cacheMu.Unlock() + entry, ok := cache[key] + if !ok || now().After(entry.expires) { + return nil, false + } + return entry.addrs, true +} + +func cacheStore(key string, addrs []netip.Addr, ttl time.Duration) { + cacheMu.Lock() + defer cacheMu.Unlock() + cache[key] = cacheEntry{addrs: addrs, expires: now().Add(ttl)} +} + +// Flush drops every cached answer. Tests use it; nothing in production does. +func Flush() { + cacheMu.Lock() + defer cacheMu.Unlock() + cache = map[string]cacheEntry{} +} + +// Lookup resolves a ".local" name to its advertised addresses. +func Lookup(ctx context.Context, name string) ([]netip.Addr, error) { + key := canonical(name) + if addrs, ok := cacheLookup(key); ok { + if len(addrs) == 0 { + return nil, fmt.Errorf("no mDNS responder for %s (cached)", name) + } + return addrs, nil + } + + addrs, ttl, err := queryAddrs(ctx, key) + if err != nil || len(addrs) == 0 { + cacheStore(key, nil, negativeTTL) + if err == nil { + err = fmt.Errorf("no mDNS responder for %s", name) + } + return nil, err + } + + cacheStore(key, addrs, ttl) + // Logged on a cache miss only, so this is at most one line per TTL per + // device rather than one per reconnect. + slog.Info("resolved host over mDNS", "host", key, "addr", addrs[0].String(), "ttl", ttl) + return addrs, nil +} + +func queryAddrs(ctx context.Context, name string) ([]netip.Addr, time.Duration, error) { + qname, err := dnsmessage.NewName(name + ".") + if err != nil { + return nil, 0, fmt.Errorf("mdns: bad name %q: %w", name, err) + } + // One packet, two questions. RFC 6762 §5.2 allows it and it saves a round + // trip on dual-stack devices. + msg := dnsmessage.Message{Questions: []dnsmessage.Question{ + {Name: qname, Type: dnsmessage.TypeA, Class: classQU}, + {Name: qname, Type: dnsmessage.TypeAAAA, Class: classQU}, + }} + packed, err := msg.Pack() + if err != nil { + return nil, 0, fmt.Errorf("mdns: pack query: %w", err) + } + + var ( + addrs []netip.Addr + ttl time.Duration + ) + err = exchange(ctx, packed, func(packet []byte) bool { + got, gotTTL, ok := parseAddrAnswer(packet, name+".") + if !ok { + return false + } + addrs, ttl = got, gotTTL + return true + }) + if err != nil { + return nil, 0, err + } + return addrs, ttl, nil +} + +// exchange sends one multicast query and feeds every reply to handle until it +// accepts one or the deadline passes. +func exchange(ctx context.Context, packed []byte, handle func([]byte) bool) error { + conn, err := listenPacket() + if err != nil { + return fmt.Errorf("mdns: open socket: %w", err) + } + defer conn.Close() + + deadline := now().Add(queryTimeout) + if d, ok := ctx.Deadline(); ok && d.Before(deadline) { + deadline = d + } + if err := conn.SetDeadline(deadline); err != nil { + return fmt.Errorf("mdns: set deadline: %w", err) + } + if _, err := conn.WriteToUDP(packed, mdnsAddr); err != nil { + return fmt.Errorf("mdns: send query: %w", err) + } + + buf := make([]byte, 1500) + for { + n, _, err := conn.ReadFromUDP(buf) + if err != nil { + return fmt.Errorf("mdns: no usable answer: %w", err) + } + if handle(buf[:n]) { + return nil + } + } +} + +func parseAddrAnswer(packet []byte, qname string) ([]netip.Addr, time.Duration, bool) { + var p dnsmessage.Parser + if _, err := p.Start(packet); err != nil { + return nil, 0, false + } + if err := p.SkipAllQuestions(); err != nil { + return nil, 0, false + } + var addrs []netip.Addr + ttl := maxTTL + // Labelled so a parse error inside the type switch abandons the whole + // packet: once the parser desynchronises, every later record is suspect. +parse: + for { + h, err := p.AnswerHeader() + if err != nil { + break parse + } + if !strings.EqualFold(h.Name.String(), qname) { + if err := p.SkipAnswer(); err != nil { + break parse + } + continue + } + switch h.Type { + case dnsmessage.TypeA: + r, err := p.AResource() + if err != nil { + break parse + } + addrs = append(addrs, netip.AddrFrom4(r.A)) + case dnsmessage.TypeAAAA: + r, err := p.AAAAResource() + if err != nil { + break parse + } + // Unmap so a v4-mapped AAAA dials as plain IPv4. + addrs = append(addrs, netip.AddrFrom16(r.AAAA).Unmap()) + default: + if err := p.SkipAnswer(); err != nil { + break parse + } + continue + } + if d := time.Duration(h.TTL) * time.Second; d < ttl { + ttl = d + } + } + return finishAnswer(addrs, ttl) +} + +func finishAnswer(addrs []netip.Addr, ttl time.Duration) ([]netip.Addr, time.Duration, bool) { + if len(addrs) == 0 { + return nil, 0, false + } + switch { + case ttl < minTTL: + ttl = minTTL + case ttl > maxTTL: + ttl = maxTTL + } + return addrs, ttl, true +} + +// Dialer dials TCP addresses, resolving ".local" host names over mDNS first. +// The embedded net.Dialer carries timeout and keep-alive; anything that is not +// a ".local" name is handed straight to it. +// +// Resolution happens per dial, not once at startup. That is what makes binding +// a device by name survive a DHCP lease change: callers that rebuild their +// connection from the original address string pick up the new IP on reconnect. +type Dialer struct { + net.Dialer +} + +// DialContext resolves address if it names a ".local" host, then dials it. +func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil || !IsLocal(host) { + return d.Dialer.DialContext(ctx, network, address) + } + + addrs, err := Lookup(ctx, host) + if err != nil { + // Name the mechanism. Without this the operator sees a bare dial + // failure and has no way to tell that resolution was the reason. + slog.Warn("mDNS resolution failed; check the device is on this LAN and the container uses host networking", + "host", host, "err", err) + return nil, fmt.Errorf("resolve %s over mDNS: %w", host, err) + } + + var firstErr error + for _, a := range addrs { + conn, err := d.Dialer.DialContext(ctx, network, net.JoinHostPort(a.String(), port)) + if err == nil { + return conn, nil + } + if firstErr == nil { + firstErr = err + } + } + return nil, fmt.Errorf("dial %s over mDNS: %w", host, firstErr) +} + +// Dial is the context-free form, for callers that have no context to pass. +func (d *Dialer) Dial(network, address string) (net.Conn, error) { + ctx := context.Background() + if d.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, d.Timeout) + defer cancel() + } + return d.DialContext(ctx, network, address) +} + +// DialContext dials with default settings. +func DialContext(ctx context.Context, network, address string) (net.Conn, error) { + var d Dialer + return d.DialContext(ctx, network, address) +} + +// DialTimeout mirrors net.DialTimeout with mDNS resolution added. +func DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + d := Dialer{Dialer: net.Dialer{Timeout: timeout}} + return d.Dial(network, address) +} diff --git a/go/internal/mdnsresolve/mdnsresolve_test.go b/go/internal/mdnsresolve/mdnsresolve_test.go new file mode 100644 index 00000000..fb9469a5 --- /dev/null +++ b/go/internal/mdnsresolve/mdnsresolve_test.go @@ -0,0 +1,265 @@ +package mdnsresolve + +import ( + "context" + "errors" + "net" + "net/netip" + "strings" + "testing" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +func TestIsLocal(t *testing.T) { + cases := []struct { + host string + want bool + }{ + {"inverter.local", true}, + {"INVERTER.LOCAL", true}, + {"inverter.local.", true}, + {"zap.local", true}, + // A literal address must never trigger a multicast query. + {"192.168.1.5", false}, + {"::1", false}, + {"example.com", false}, + {"localhost", false}, + {"local", false}, + {"notlocal", false}, + {"", false}, + } + for _, c := range cases { + if got := IsLocal(c.host); got != c.want { + t.Errorf("IsLocal(%q) = %v, want %v", c.host, got, c.want) + } + } +} + +func aResource(t *testing.T, name string, ip [4]byte, ttl uint32) dnsmessage.Resource { + t.Helper() + return dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: mustDNSName(t, name), Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, TTL: ttl, + }, + Body: &dnsmessage.AResource{A: ip}, + } +} + +func packAnswer(t *testing.T, qname string, answers []dnsmessage.Resource) []byte { + t.Helper() + msg := dnsmessage.Message{ + Header: dnsmessage.Header{Response: true, Authoritative: true}, + Questions: []dnsmessage.Question{{Name: mustDNSName(t, qname), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, + Answers: answers, + } + packet, err := msg.Pack() + if err != nil { + t.Fatalf("pack: %v", err) + } + return packet +} + +func TestParseAddrAnswer(t *testing.T) { + qname := "inverter.local." + packet := packAnswer(t, qname, []dnsmessage.Resource{aResource(t, qname, [4]byte{192, 168, 1, 42}, 60)}) + + addrs, ttl, ok := parseAddrAnswer(packet, qname) + if !ok { + t.Fatal("parseAddrAnswer did not accept a valid answer") + } + if len(addrs) != 1 || addrs[0].String() != "192.168.1.42" { + t.Fatalf("addrs = %v, want [192.168.1.42]", addrs) + } + if ttl != 60*time.Second { + t.Fatalf("ttl = %v, want 60s", ttl) + } + + // An answer for a different name must be ignored. + if _, _, ok := parseAddrAnswer(packet, "other.local."); ok { + t.Fatal("accepted an answer for a different name") + } + // Garbage must not panic or resolve. + if _, _, ok := parseAddrAnswer([]byte{1, 2, 3}, qname); ok { + t.Fatal("accepted a malformed packet") + } +} + +func TestParseAddrAnswerClampsTTL(t *testing.T) { + qname := "inverter.local." + for _, c := range []struct { + name string + ttl uint32 + want time.Duration + }{ + // A device advertising a 1 s TTL must not make every Modbus reconnect + // re-query the LAN. + {"below floor", 1, minTTL}, + // A very long TTL must not outlive a DHCP move. + {"above ceiling", 86400, maxTTL}, + {"inside range", 90, 90 * time.Second}, + } { + t.Run(c.name, func(t *testing.T) { + packet := packAnswer(t, qname, []dnsmessage.Resource{aResource(t, qname, [4]byte{10, 0, 0, 1}, c.ttl)}) + _, ttl, ok := parseAddrAnswer(packet, qname) + if !ok { + t.Fatal("answer rejected") + } + if ttl != c.want { + t.Fatalf("ttl = %v, want %v", ttl, c.want) + } + }) + } +} + +// startResponder points the package at a loopback UDP socket that answers one +// query, so the real send/parse path is exercised without touching the LAN. +func startResponder(t *testing.T, answers []dnsmessage.Resource) { + t.Helper() + rc, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + if err != nil { + t.Fatalf("listen responder: %v", err) + } + + origAddr, origListen := mdnsAddr, listenPacket + mdnsAddr = rc.LocalAddr().(*net.UDPAddr) + listenPacket = func() (*net.UDPConn, error) { + return net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + } + + done := make(chan struct{}) + go func() { + defer close(done) + buf := make([]byte, 1500) + _ = rc.SetReadDeadline(time.Now().Add(2 * time.Second)) + n, from, err := rc.ReadFromUDP(buf) + if err != nil { + return + } + if answers == nil { + return // silent responder: exercises the negative path + } + var p dnsmessage.Parser + hdr, err := p.Start(buf[:n]) + if err != nil { + return + } + q, err := p.Question() + if err != nil { + return + } + resp := dnsmessage.Message{ + Header: dnsmessage.Header{ID: hdr.ID, Response: true, Authoritative: true}, + Questions: []dnsmessage.Question{q}, + Answers: answers, + } + packed, err := resp.Pack() + if err != nil { + return + } + _, _ = rc.WriteToUDP(packed, from) + }() + + t.Cleanup(func() { + _ = rc.Close() + <-done + mdnsAddr, listenPacket = origAddr, origListen + Flush() + }) +} + +func TestLookupResolvesLocalName(t *testing.T) { + Flush() + startResponder(t, []dnsmessage.Resource{aResource(t, "inverter.local.", [4]byte{192, 168, 1, 42}, 60)}) + + addrs, err := Lookup(context.Background(), "inverter.local") + if err != nil { + t.Fatalf("Lookup: %v", err) + } + if len(addrs) != 1 || addrs[0] != netip.MustParseAddr("192.168.1.42") { + t.Fatalf("addrs = %v, want [192.168.1.42]", addrs) + } + + // The answer must now be cached: a second call cannot need the responder, + // which has already stopped. + again, err := Lookup(context.Background(), "INVERTER.local") + if err != nil { + t.Fatalf("cached Lookup: %v", err) + } + if len(again) != 1 || again[0] != addrs[0] { + t.Fatalf("cached addrs = %v, want %v", again, addrs) + } +} + +func TestLookupCachesNegativeAnswer(t *testing.T) { + Flush() + startResponder(t, nil) // never answers + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if _, err := Lookup(ctx, "missing.local"); err == nil { + t.Fatal("expected a lookup failure when nothing answers") + } + + addrs, ok := cacheLookup("missing.local") + if !ok { + t.Fatal("a failed lookup should be negatively cached") + } + if len(addrs) != 0 { + t.Fatalf("negative cache holds %v, want no addresses", addrs) + } +} + +func TestCacheExpires(t *testing.T) { + Flush() + base := time.Now() + orig := now + now = func() time.Time { return base } + t.Cleanup(func() { now = orig; Flush() }) + + cacheStore("inverter.local", []netip.Addr{netip.MustParseAddr("192.168.1.9")}, 30*time.Second) + if _, ok := cacheLookup("inverter.local"); !ok { + t.Fatal("entry should be live immediately after store") + } + + now = func() time.Time { return base.Add(31 * time.Second) } + if _, ok := cacheLookup("inverter.local"); ok { + t.Fatal("entry should have expired") + } +} + +func TestDialerSkipsResolutionForPlainHosts(t *testing.T) { + orig := listenPacket + listenPacket = func() (*net.UDPConn, error) { + t.Error("issued an mDNS query for a host that is not a .local name") + return nil, errors.New("should not be called") + } + t.Cleanup(func() { listenPacket = orig }) + + d := Dialer{Dialer: net.Dialer{Timeout: 500 * time.Millisecond}} + // Nothing listens on port 1; the point is that the failure comes from the + // dial, not from resolution. + if _, err := d.Dial("tcp", "127.0.0.1:1"); err == nil { + t.Fatal("expected the dial to fail") + } else if strings.Contains(err.Error(), "mDNS") { + t.Fatalf("plain IP dial went through mDNS: %v", err) + } +} + +func TestDialerReportsResolutionFailure(t *testing.T) { + Flush() + startResponder(t, nil) // never answers + + d := Dialer{Dialer: net.Dialer{Timeout: 100 * time.Millisecond}} + _, err := d.Dial("tcp", "missing.local:502") + if err == nil { + t.Fatal("expected a failure") + } + // The error must name the mechanism — an operator reading the log has to be + // able to tell resolution apart from an unreachable device. + if !strings.Contains(err.Error(), "mDNS") { + t.Fatalf("error %q does not mention mDNS", err) + } +} diff --git a/go/internal/scanner/mdns.go b/go/internal/mdnsresolve/reverse.go similarity index 51% rename from go/internal/scanner/mdns.go rename to go/internal/mdnsresolve/reverse.go index 4963ed52..b1206136 100644 --- a/go/internal/scanner/mdns.go +++ b/go/internal/mdnsresolve/reverse.go @@ -1,20 +1,19 @@ -package scanner +package mdnsresolve import ( "context" "fmt" "net" "strings" - "time" "golang.org/x/net/dns/dnsmessage" ) -var mdnsAddr = &net.UDPAddr{IP: net.IPv4(224, 0, 0, 251), Port: 5353} - -// reverseMDNS sends a reverse PTR query with the QU bit set so devices reply -// directly to our ephemeral socket instead of requiring a bind to port 5353. -func reverseMDNS(ctx context.Context, ip string) string { +// ReverseLookup asks the device at ip to name itself, so a discovered device +// can be shown and stored by its self-broadcast ".local" name rather than a +// DHCP-assigned address. Returns "" when nothing answers — callers treat a +// missing name as ordinary, not as an error. +func ReverseLookup(ctx context.Context, ip string) string { v4 := net.ParseIP(ip).To4() if v4 == nil { return "" @@ -25,35 +24,19 @@ func reverseMDNS(ctx context.Context, ip string) string { return "" } msg := dnsmessage.Message{Questions: []dnsmessage.Question{{ - Name: name, Type: dnsmessage.TypePTR, Class: dnsmessage.Class(0x8001), + Name: name, Type: dnsmessage.TypePTR, Class: classQU, }}} packed, err := msg.Pack() if err != nil { return "" } - conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) - if err != nil { - return "" - } - defer conn.Close() - deadline := time.Now().Add(900 * time.Millisecond) - if d, ok := ctx.Deadline(); ok && d.Before(deadline) { - deadline = d - } - _ = conn.SetDeadline(deadline) - if _, err := conn.WriteToUDP(packed, mdnsAddr); err != nil { - return "" - } - buf := make([]byte, 1500) - for { - n, _, err := conn.ReadFromUDP(buf) - if err != nil { - return "" - } - if host := parsePTRAnswer(buf[:n], qname); host != "" { - return host - } - } + + var host string + _ = exchange(ctx, packed, func(packet []byte) bool { + host = parsePTRAnswer(packet, qname) + return host != "" + }) + return host } func parsePTRAnswer(packet []byte, qname string) string { diff --git a/go/internal/scanner/mdns_test.go b/go/internal/mdnsresolve/reverse_test.go similarity index 98% rename from go/internal/scanner/mdns_test.go rename to go/internal/mdnsresolve/reverse_test.go index 5a77c8f6..af8801bc 100644 --- a/go/internal/scanner/mdns_test.go +++ b/go/internal/mdnsresolve/reverse_test.go @@ -1,4 +1,4 @@ -package scanner +package mdnsresolve import ( "testing" diff --git a/go/internal/modbus/tcp_client.go b/go/internal/modbus/tcp_client.go index 78b7e7ba..b5b790ab 100644 --- a/go/internal/modbus/tcp_client.go +++ b/go/internal/modbus/tcp_client.go @@ -7,6 +7,8 @@ import ( "io" "net" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) const ( @@ -39,10 +41,13 @@ func newTCPClient(addr string, timeout, keepAlive time.Duration) *tcpClient { } func (c *tcpClient) Open() error { - dialer := net.Dialer{ + // Resolution happens here, on every Open, so a device configured by its + // ".local" name is found again after a DHCP lease moves it — the reconnect + // path rebuilds the client from c.addr and picks up the new address. + dialer := mdnsresolve.Dialer{Dialer: net.Dialer{ Timeout: modbusDialTimeout, KeepAlive: c.keepAlive, - } + }} conn, err := dialer.Dial("tcp", c.addr) if err != nil { return err diff --git a/go/internal/mqtt/client.go b/go/internal/mqtt/client.go index fb1ec167..a711dd96 100644 --- a/go/internal/mqtt/client.go +++ b/go/internal/mqtt/client.go @@ -4,12 +4,15 @@ package mqtt import ( "fmt" "log/slog" + "net" + "net/url" "sync" "time" paho "github.com/eclipse/paho.mqtt.golang" "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // Capability wraps a paho client to match drivers.MQTTCap. @@ -51,6 +54,14 @@ func Dial(host string, port int, username, password, clientID string) (*Capabili } opts := paho.NewClientOptions(). AddBroker(fmt.Sprintf("tcp://%s:%d", host, port)). + // paho's built-in dialer goes through the stdlib resolver, which never + // answers a ".local" name. Every broker URL built here is tcp://, so a + // TCP-only replacement is complete; non-".local" hosts fall through to + // a plain dial inside mdnsresolve. + SetCustomOpenConnectionFn(func(uri *url.URL, o paho.ClientOptions) (net.Conn, error) { + d := mdnsresolve.Dialer{Dialer: net.Dialer{Timeout: o.ConnectTimeout}} + return d.Dial("tcp", uri.Host) + }). SetClientID(clientID). SetAutoReconnect(true). SetConnectRetry(true). diff --git a/go/internal/scanner/scanner.go b/go/internal/scanner/scanner.go index 0d8dea31..612d8207 100644 --- a/go/internal/scanner/scanner.go +++ b/go/internal/scanner/scanner.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "time" + + "github.com/srcfl/ftw/go/internal/mdnsresolve" ) // FoundDevice is one open port discovered on the local network. @@ -129,7 +131,7 @@ func resolveHostnames(ctx context.Context, devices []FoundDevice) { } if name == "" && ctx.Err() == nil { mdnsCtx, cancel := context.WithTimeout(ctx, 900*time.Millisecond) - name = reverseMDNS(mdnsCtx, ip) + name = mdnsresolve.ReverseLookup(mdnsCtx, ip) cancel() } if name != "" { From 8669d0ca7bcb9b5cadde3e538869bb3c22c98346 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Fri, 31 Jul 2026 14:46:14 +0200 Subject: [PATCH 2/2] build: share one alpine base between core and the updater @ The updater sidecar ran docker:27-cli. That is alpine-based too, but it is a different alpine on its own cadence, so the deployment pulled two rootfs blobs and tracked two upgrade streams for what looked like one distribution. Put the sidecar on the same alpine:3.22 tag and copy the docker CLI and compose plugin out of the official image instead -- same artifact, version pinned, no package repository added, and both are static Go binaries so neither cares about libc. The optimizer cannot follow. Measured on python:3.12-alpine: cvxpy publishes no musllinux wheel at all (newest is 0.4.10), and clarabel, osqp and ecos have none either. Source builds mean compiling a Rust solver and two C++ solvers on every release, arm64 under emulation included, and the attempt fails in sparsediffpy before reaching them. It stays on Debian slim but leaves oldstable for trixie. Make the boundary test assert all of that. It grepped one file for ^FROM alpine: with no error message, which proved half the invariant and said nothing when it broke. Co-authored-by: HuggeK <48095810+HuggeK@users.noreply.github.com> @ --- .changeset/one-alpine-base.md | 32 ++++++++++++++++++++++++++++ Dockerfile | 14 ++++++++++++ Dockerfile.optimizer | 21 ++++++++++++++++-- Dockerfile.updater | 27 ++++++++++++++++++----- go/cmd/ftw/main.go | 7 ++++++ optimizer/pyproject.toml | 2 +- scripts/test-container-boundaries.sh | 31 ++++++++++++++++++++++++++- 7 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 .changeset/one-alpine-base.md diff --git a/.changeset/one-alpine-base.md b/.changeset/one-alpine-base.md new file mode 100644 index 00000000..a6044497 --- /dev/null +++ b/.changeset/one-alpine-base.md @@ -0,0 +1,32 @@ +--- +"ftw": minor +--- + +Run core and the updater sidecar on one shared base layer, and move the optimizer off oldstable. + +The updater sidecar was `docker:27-cli`. That image is alpine-based too, but it +is a *different* alpine — its own base layer on its own cadence — so a host +pulled two rootfs blobs and tracked two upgrade streams for what looked like one +distribution. The sidecar now sits on the same `alpine:3.22` tag as core, with +the `docker` CLI and the compose plugin copied straight out of the official CLI +image. That is the same upstream artifact, with the version pinned explicitly +and no package repository added; both are statically linked Go binaries, so +neither depends on the base's libc. + +The third image cannot join them. CVXPY publishes no musllinux wheels — the +newest that exists is 0.4.10 — and neither do clarabel, osqp or ecos, so the +optimizer cannot be built on alpine without compiling a Rust solver and two C++ +solvers from source on every release, arm64 under emulation included. It stays +on Debian slim, but moves from bookworm to trixie, because bookworm is oldstable +and part of the deployment was already off full security support. The container +boundary test now asserts all of this instead of grepping one file for a fixed +string with no error message. + +The optimizer is versioned independently and moves to 1.4.0: its image is a +materially different artifact once the base changes, and its release workflow +verifies that a published image's revision label matches the commit it claims, +so the new base could not ship under the old version number. + +The zoneinfo database is also embedded in the core binary as a fallback. It was +already installed in the image; the point is that a base without it can no +longer silently push `time.Local` to UTC and mis-time price and plan windows. diff --git a/Dockerfile b/Dockerfile index 6526349c..88db8e93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,10 +36,24 @@ RUN cd go && \ go build -trimpath -ldflags="-s -w -X main.Version=${VERSION}" \ -o /out/ftw-backup ./cmd/ftw-backup # --- Runtime --------------------------------------------------------------- +# alpine:3.22, and Dockerfile.updater now uses the same tag, so the two images +# a host runs side by side share one base layer instead of pulling two. +# +# The binary is CGO_ENABLED=0 and fully static, so this picks the image's +# *userland*, not a libc the process depends on. That is why the choice is +# reversible and why it has no bearing on .local resolution — see +# go/internal/mdnsresolve. +# +# Dockerfile.optimizer cannot join this base: CVXPY publishes no musllinux +# wheels at all, so see the note there before trying to unify the third image. FROM alpine:3.22 # HTTPS integrations and timezone-aware price/plan windows need these at # runtime. BusyBox wget provides the health check without adding Python/curl. +# +# tzdata is belt-and-braces now: the binary also embeds the zoneinfo database +# (see the time/tzdata import in cmd/ftw). Without either, time.Local silently +# degrades to UTC and mis-times price and plan windows with no error at all. RUN apk add --no-cache ca-certificates tzdata # Image layout: diff --git a/Dockerfile.optimizer b/Dockerfile.optimizer index f8847321..3950a39c 100644 --- a/Dockerfile.optimizer +++ b/Dockerfile.optimizer @@ -1,11 +1,28 @@ # Independently releasable FTW mathematical optimizer. -FROM python:3.12-slim-bookworm AS build +# +# This image does NOT share a base with core and the updater, and cannot. +# Measured, not assumed, on python:3.12-alpine: +# +# pip install --only-binary=:all: cvxpy==1.9.2 +# → "Could not find a version that satisfies the requirement +# cvxpy==1.9.2 (from versions: 0.4.10)" +# +# CVXPY publishes no musllinux wheels — 0.4.10 is the newest that exists — and +# clarabel, osqp and ecos have none either. numpy, scipy and highspy do. Falling +# back to source builds means compiling a Rust solver and two C++ solvers on +# every release, for arm64 under emulation as well; attempted, and it fails in +# sparsediffpy's build backend before it even reaches them. +# +# So the stack runs two bases by necessity. What is fixed here is the suite: +# bookworm is oldstable, so this moves to trixie to match core's security +# cadence even though the layers stay separate. +FROM python:3.12-slim-trixie AS build COPY optimizer/ /src/optimizer/ RUN python -m venv /opt/venv && \ /opt/venv/bin/pip install --no-cache-dir /src/optimizer -FROM python:3.12-slim-bookworm +FROM python:3.12-slim-trixie ARG VERSION=dev ARG BUILD_SHA="" diff --git a/Dockerfile.updater b/Dockerfile.updater index e00b5d5c..50ff1139 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -28,12 +28,29 @@ RUN cd go && \ -o /out/ftw-updater ./cmd/ftw-updater # --- Runtime --------------------------------------------------------------- -# docker:27-cli bundles the `docker` binary + `docker compose` plugin + -# ca-certificates + tzdata, which is exactly what we need to execute -# compose pulls against the host daemon. -FROM docker:27-cli +# The same alpine:3.22 rootfs as Dockerfile, so core and the sidecar share one +# base layer: a host pulls it once instead of twice, and there is one suite to +# track for both. +# +# This was docker:27-cli. That image is alpine-based too, but it is a +# *different* alpine — its own base layer, its own upgrade cadence — so it +# shared nothing with core in practice. +# +# The docker CLI and the compose plugin are copied straight out of the official +# CLI image instead: same upstream artifact, version pinned explicitly, and no +# package repository added. Both are statically linked Go binaries (verified: +# musl's loader reports "Not a valid dynamic program", and both run correctly +# once copied), so nothing here depends on which libc the base ships. +FROM alpine:3.22 + +# docker:27-cli bundled these; on a bare base they have to be asked for. +# Compose pull hits GHCR over TLS. +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker:27-cli /usr/local/libexec/docker/cli-plugins/docker-compose \ + /usr/local/libexec/docker/cli-plugins/docker-compose -# Compose pull hits GHCR over TLS; docker:cli already bundles ca-certs. COPY --from=builder /out/ftw-updater /usr/local/bin/ftw-updater COPY LICENSE NOTICE /usr/share/doc/ftw/ diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..cfe68949 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -24,6 +24,13 @@ import ( "sync" "syscall" "time" + // Embed the zoneinfo database in the binary. Production code uses + // time.Local, which silently falls back to UTC when the runtime has no + // zoneinfo tree — no error, no log line, just price windows and plan + // boundaries an hour or two out. The system database still wins when it is + // present; this only removes the silent-failure mode if a base image ever + // stops shipping tzdata. + _ "time/tzdata" "github.com/srcfl/ftw/go/internal/api" "github.com/srcfl/ftw/go/internal/arp" diff --git a/optimizer/pyproject.toml b/optimizer/pyproject.toml index 2611fca7..8ef2a8de 100644 --- a/optimizer/pyproject.toml +++ b/optimizer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ftw-optimizer" -version = "1.3.2" +version = "1.4.0" description = "CVXPY planning engine for ftw" requires-python = ">=3.11" dependencies = [ diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index ea827f6b..ce871a7b 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -9,7 +9,36 @@ if grep -Eq 'COPY optimizer/|--from=optimizer|/opt/venv|FTW_OPTIMIZER_(PYTHON|DI exit 1 fi -grep -q '^FROM alpine:' Dockerfile +# Core and the updater are meant to share one base layer, so a host pulls that +# rootfs once. Comparing the two runtime stages is what actually holds that; +# grepping for a fixed string in one file only ever proved half of it, and did +# so without printing anything when it failed. +core_base=$(grep -E '^FROM ' Dockerfile | tail -1 | awk '{print $2}') +updater_base=$(grep -E '^FROM ' Dockerfile.updater | tail -1 | awk '{print $2}') + +case "${core_base}" in + alpine:*) ;; + *) + echo "Dockerfile runtime stage is '${core_base}', expected an alpine: tag" >&2 + exit 1 + ;; +esac + +if [ "${core_base}" != "${updater_base}" ]; then + echo "core and updater runtime bases differ: '${core_base}' vs '${updater_base}'" >&2 + echo "they must be the same tag or the shared base layer silently stops being shared" >&2 + exit 1 +fi + +# The optimizer is knowingly on a different base: CVXPY publishes no musllinux +# wheels, so it cannot follow the other two onto alpine. Assert that rather than +# leave it looking like an oversight someone should "fix". +if grep -qE '^FROM python:[^ ]*alpine' Dockerfile.optimizer; then + echo "Dockerfile.optimizer is on an alpine base, which cannot resolve CVXPY" >&2 + echo "cvxpy has no musllinux wheel (newest is 0.4.10); see the note in that file" >&2 + exit 1 +fi + grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile